Hugging Face's logo

jtatman
/
tinymistral-mediqa-248m

Text Generation
Transformers
Safetensors
mistral
medical
text-generation-inference
Inference Endpoints
Model card Files Community
1
jtatman commited on
Commit
6973c31
•
1 Parent(s): 0828b01

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +0 -44
  2. tokenizer.json +0 -45
  3. tokenizer_config.json +1 -48
special_tokens_map.json CHANGED
@@ -1,48 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- {
4
- "content": "<|im_start|>user",
5
- "lstrip": false,
6
- "normalized": false,
7
- "rstrip": false,
8
- "single_word": false
9
- },
10
- {
11
- "content": "<|im_end|>",
12
- "lstrip": false,
13
- "normalized": false,
14
- "rstrip": false,
15
- "single_word": false
16
- },
17
- {
18
- "content": "<|im_start|>assistant",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- },
24
- {
25
- "content": "<|im_start|>system",
26
- "lstrip": false,
27
- "normalized": false,
28
- "rstrip": false,
29
- "single_word": false
30
- },
31
- {
32
- "content": "<|endoftext|>",
33
- "lstrip": false,
34
- "normalized": false,
35
- "rstrip": false,
36
- "single_word": false
37
- },
38
- {
39
- "content": "<|im_start|>",
40
- "lstrip": false,
41
- "normalized": false,
42
- "rstrip": false,
43
- "single_word": false
44
- }
45
- ],
46
  "bos_token": {
47
  "content": "<|bos|>",
48
  "lstrip": false,
 
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "bos_token": {
3
  "content": "<|bos|>",
4
  "lstrip": false,
tokenizer.json CHANGED
@@ -88,51 +88,6 @@
88
  "rstrip": false,
89
  "normalized": false,
90
  "special": true
91
- },
92
- {
93
- "id": 32005,
94
- "content": "<|im_start|>user",
95
- "single_word": false,
96
- "lstrip": false,
97
- "rstrip": false,
98
- "normalized": false,
99
- "special": true
100
- },
101
- {
102
- "id": 32006,
103
- "content": "<|im_end|>",
104
- "single_word": false,
105
- "lstrip": false,
106
- "rstrip": false,
107
- "normalized": false,
108
- "special": true
109
- },
110
- {
111
- "id": 32007,
112
- "content": "<|im_start|>assistant",
113
- "single_word": false,
114
- "lstrip": false,
115
- "rstrip": false,
116
- "normalized": false,
117
- "special": true
118
- },
119
- {
120
- "id": 32008,
121
- "content": "<|im_start|>system",
122
- "single_word": false,
123
- "lstrip": false,
124
- "rstrip": false,
125
- "normalized": false,
126
- "special": true
127
- },
128
- {
129
- "id": 32009,
130
- "content": "<|im_start|>",
131
- "single_word": false,
132
- "lstrip": false,
133
- "rstrip": false,
134
- "normalized": false,
135
- "special": true
136
  }
137
  ],
138
  "normalizer": {
 
88
  "rstrip": false,
89
  "normalized": false,
90
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  }
92
  ],
93
  "normalizer": {
tokenizer_config.json CHANGED
@@ -65,56 +65,9 @@
65
  "rstrip": false,
66
  "single_word": false,
67
  "special": true
68
- },
69
- "32005": {
70
- "content": "<|im_start|>user",
71
- "lstrip": false,
72
- "normalized": false,
73
- "rstrip": false,
74
- "single_word": false,
75
- "special": true
76
- },
77
- "32006": {
78
- "content": "<|im_end|>",
79
- "lstrip": false,
80
- "normalized": false,
81
- "rstrip": false,
82
- "single_word": false,
83
- "special": true
84
- },
85
- "32007": {
86
- "content": "<|im_start|>assistant",
87
- "lstrip": false,
88
- "normalized": false,
89
- "rstrip": false,
90
- "single_word": false,
91
- "special": true
92
- },
93
- "32008": {
94
- "content": "<|im_start|>system",
95
- "lstrip": false,
96
- "normalized": false,
97
- "rstrip": false,
98
- "single_word": false,
99
- "special": true
100
- },
101
- "32009": {
102
- "content": "<|im_start|>",
103
- "lstrip": false,
104
- "normalized": false,
105
- "rstrip": false,
106
- "single_word": false,
107
- "special": true
108
  }
109
  },
110
- "additional_special_tokens": [
111
- "<|im_start|>user",
112
- "<|im_end|>",
113
- "<|im_start|>assistant",
114
- "<|im_start|>system",
115
- "<|endoftext|>",
116
- "<|im_start|>"
117
- ],
118
  "bos_token": "<|bos|>",
119
  "clean_up_tokenization_spaces": false,
120
  "eos_token": "<|endoftext|>",
 
65
  "rstrip": false,
66
  "single_word": false,
67
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  }
69
  },
70
+ "additional_special_tokens": [],
 
 
 
 
 
 
 
71
  "bos_token": "<|bos|>",
72
  "clean_up_tokenization_spaces": false,
73
  "eos_token": "<|endoftext|>",