ybelkada commited on
Commit
cb14093
1 Parent(s): 03ffc82

Upload processor

Browse files
added_tokens.json CHANGED
@@ -1,3 +1,4 @@
1
  {
2
- "<image>": 32000
 
3
  }
 
1
  {
2
+ "<image>": 32000,
3
+ "<pad>": 32001
4
  }
special_tokens_map.json CHANGED
@@ -13,6 +13,7 @@
13
  "rstrip": false,
14
  "single_word": false
15
  },
 
16
  "unk_token": {
17
  "content": "<unk>",
18
  "lstrip": false,
 
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
+ "pad_token": "<pad>",
17
  "unk_token": {
18
  "content": "<unk>",
19
  "lstrip": false,
tokenizer.json CHANGED
@@ -37,7 +37,16 @@
37
  "lstrip": false,
38
  "rstrip": false,
39
  "normalized": false,
40
- "special": false
 
 
 
 
 
 
 
 
 
41
  }
42
  ],
43
  "normalizer": {
 
37
  "lstrip": false,
38
  "rstrip": false,
39
  "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 32001,
44
+ "content": "<pad>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
  }
51
  ],
52
  "normalizer": {
tokenizer_config.json CHANGED
@@ -33,6 +33,14 @@
33
  "rstrip": false,
34
  "single_word": false,
35
  "special": true
 
 
 
 
 
 
 
 
36
  }
37
  },
38
  "bos_token": "<s>",
@@ -40,11 +48,12 @@
40
  "eos_token": "</s>",
41
  "legacy": false,
42
  "model_max_length": 1000000000000000019884624838656,
43
- "pad_token": null,
44
  "padding_side": "left",
45
  "processor_class": "LlavaProcessor",
46
  "sp_model_kwargs": {},
47
  "tokenizer_class": "LlamaTokenizer",
 
48
  "unk_token": "<unk>",
49
  "use_default_system_prompt": false
50
  }
 
33
  "rstrip": false,
34
  "single_word": false,
35
  "special": true
36
+ },
37
+ "32001": {
38
+ "content": "<pad>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
  }
45
  },
46
  "bos_token": "<s>",
 
48
  "eos_token": "</s>",
49
  "legacy": false,
50
  "model_max_length": 1000000000000000019884624838656,
51
+ "pad_token": "<pad>",
52
  "padding_side": "left",
53
  "processor_class": "LlavaProcessor",
54
  "sp_model_kwargs": {},
55
  "tokenizer_class": "LlamaTokenizer",
56
+ "trust_remote_code": false,
57
  "unk_token": "<unk>",
58
  "use_default_system_prompt": false
59
  }