RaushanTurganbay HF staff commited on
Commit
c2de364
1 Parent(s): 0524afe

Upload processor

Browse files
preprocessor_config.json CHANGED
@@ -7,6 +7,7 @@
7
  "do_center_crop": true,
8
  "do_convert_rgb": true,
9
  "do_normalize": true,
 
10
  "do_rescale": true,
11
  "do_resize": true,
12
  "image_grid_pinpoints": [
 
7
  "do_center_crop": true,
8
  "do_convert_rgb": true,
9
  "do_normalize": true,
10
+ "do_pad": true,
11
  "do_rescale": true,
12
  "do_resize": true,
13
  "image_grid_pinpoints": [
processor_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "image_token": "<image>",
3
+ "patch_size": 14,
4
+ "processor_class": "LlavaNextProcessor",
5
+ "vision_feature_select_strategy": "default"
6
+ }
tokenizer.json CHANGED
@@ -1,14 +1,7 @@
1
  {
2
  "version": "1.0",
3
  "truncation": null,
4
- "padding": {
5
- "strategy": "BatchLongest",
6
- "direction": "Right",
7
- "pad_to_multiple_of": null,
8
- "pad_id": 0,
9
- "pad_type_id": 0,
10
- "pad_token": "<unk>"
11
- },
12
  "added_tokens": [
13
  {
14
  "id": 0,
@@ -47,23 +40,13 @@
47
  "special": true
48
  }
49
  ],
50
- "normalizer": {
51
- "type": "Sequence",
52
- "normalizers": [
53
- {
54
- "type": "Prepend",
55
- "prepend": "▁"
56
- },
57
- {
58
- "type": "Replace",
59
- "pattern": {
60
- "String": " "
61
- },
62
- "content": "▁"
63
- }
64
- ]
65
  },
66
- "pre_tokenizer": null,
67
  "post_processor": {
68
  "type": "TemplateProcessing",
69
  "single": [
@@ -150,6 +133,7 @@
150
  "end_of_word_suffix": null,
151
  "fuse_unk": true,
152
  "byte_fallback": true,
 
153
  "vocab": {
154
  "<unk>": 0,
155
  "<s>": 1,
 
1
  {
2
  "version": "1.0",
3
  "truncation": null,
4
+ "padding": null,
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
 
40
  "special": true
41
  }
42
  ],
43
+ "normalizer": null,
44
+ "pre_tokenizer": {
45
+ "type": "Metaspace",
46
+ "replacement": "▁",
47
+ "prepend_scheme": "first",
48
+ "split": false
 
 
 
 
 
 
 
 
 
49
  },
 
50
  "post_processor": {
51
  "type": "TemplateProcessing",
52
  "single": [
 
133
  "end_of_word_suffix": null,
134
  "fuse_unk": true,
135
  "byte_fallback": true,
136
+ "ignore_merges": false,
137
  "vocab": {
138
  "<unk>": 0,
139
  "<s>": 1,