Hugging Face's logo

CHIH-HUNG
/
Mistral-7B-v0.1-Open-Platypus_2.5w-r16-gate_up_down

Text Generation
Transformers
PyTorch
mistral
text-generation-inference
Inference Endpoints
Model card Files Community
CHIH-HUNG commited on
Commit
b9d309e
•
1 Parent(s): fc668a8

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +5 -0
  2. special_tokens_map.json +8 -21
  3. tokenizer_config.json +5 -1
added_tokens.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "</s>": 2,
3
+ "<s>": 1,
4
+ "<unk>": 0
5
+ }
special_tokens_map.json CHANGED
@@ -1,23 +1,10 @@
1
  {
2
- "bos_token": {
3
- "content": "<s>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "</s>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "unk_token": {
17
- "content": "<unk>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- }
23
  }
 
1
  {
2
+ "additional_special_tokens": [
3
+ "<unk>",
4
+ "<s>",
5
+ "</s>"
6
+ ],
7
+ "bos_token": "<s>",
8
+ "eos_token": "</s>",
9
+ "unk_token": "<unk>"
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  }
tokenizer_config.json CHANGED
@@ -25,7 +25,11 @@
25
  "special": true
26
  }
27
  },
28
- "additional_special_tokens": [],
 
 
 
 
29
  "bos_token": "<s>",
30
  "clean_up_tokenization_spaces": false,
31
  "eos_token": "</s>",
 
25
  "special": true
26
  }
27
  },
28
+ "additional_special_tokens": [
29
+ "<unk>",
30
+ "<s>",
31
+ "</s>"
32
+ ],
33
  "bos_token": "<s>",
34
  "clean_up_tokenization_spaces": false,
35
  "eos_token": "</s>",