File size: 314 Bytes
744618f
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
{
    "additional_special_tokens": [
        "<|fim_begin|>",
        "<|fim_hole|>",
        "<|fim_end|>",
        "<|fim_pad|>",
        "<|repo_name|>",
        "<|file_sep|>"
    ],
    "bos_token": "<|startoftext|>",
    "cls_token": "[CLS]",
    "eos_token": "<|endoftext|>",
    "gmask_token": "[gMASK]"
}