Hugging Face's logo

not-lain
/
smol-gec-sft

Text Generation
Transformers
Safetensors
llama
trl
sft
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
not-lain commited on
Commit
b02f034
•
1 Parent(s): 85a34e1

Update special_tokens_map.json

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +18 -39
special_tokens_map.json CHANGED
@@ -1,44 +1,23 @@
1
  {
2
  "additional_special_tokens": [
3
- "<|endoftext|>",
4
- "<|im_start|>",
5
- "<|im_end|>",
6
- "<repo_name>",
7
- "<reponame>",
8
- "<file_sep>",
9
- "<filename>",
10
- "<gh_stars>",
11
- "<issue_start>",
12
- "<issue_comment>",
13
- "<issue_closed>",
14
- "<jupyter_start>",
15
- "<jupyter_text>",
16
- "<jupyter_code>",
17
- "<jupyter_output>",
18
- "<jupyter_script>",
19
- "<empty_output>"
20
  ],
21
- "bos_token": {
22
- "content": "<|endoftext|>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false
27
- },
28
- "eos_token": {
29
- "content": "<|endoftext|>",
30
- "lstrip": false,
31
- "normalized": false,
32
- "rstrip": false,
33
- "single_word": false
34
- },
35
- "pad_token": {
36
- "content": "<|endoftext|>",
37
- "lstrip": false,
38
- "normalized": false,
39
- "rstrip": false,
40
- "single_word": false
41
- },
42
  "unk_token": {
43
  "content": "<|endoftext|>",
44
  "lstrip": false,
@@ -46,4 +25,4 @@
46
  "rstrip": false,
47
  "single_word": false
48
  }
49
- }
 
1
  {
2
  "additional_special_tokens": [
3
+ {
4
+ "content": "<|im_start|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ {
11
+ "content": "<|im_end|>",
12
+ "lstrip": false,
13
+ "normalized": false,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ }
 
 
 
17
  ],
18
+ "bos_token": "<|im_start|>",
19
+ "eos_token": "<|im_end|>",
20
+ "pad_token": "<|im_end|>",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  "unk_token": {
22
  "content": "<|endoftext|>",
23
  "lstrip": false,
 
25
  "rstrip": false,
26
  "single_word": false
27
  }
28
+ }