upload tokenizer
Browse files- cvlm_llama2_tokenizer_100img_and_224loc_addpatch/added_tokens.json +332 -0
- cvlm_llama2_tokenizer_100img_and_224loc_addpatch/special_tokens_map.json +24 -0
- cvlm_llama2_tokenizer_100img_and_224loc_addpatch/tokenizer.model +3 -0
- cvlm_llama2_tokenizer_100img_and_224loc_addpatch/tokenizer_config.json +34 -0
cvlm_llama2_tokenizer_100img_and_224loc_addpatch/added_tokens.json
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"</img>": 32101,
|
| 3 |
+
"</patch>": 32103,
|
| 4 |
+
"<box_end>": 32329,
|
| 5 |
+
"<box_start>": 32328,
|
| 6 |
+
"<img>": 32100,
|
| 7 |
+
"<img_00000>": 32000,
|
| 8 |
+
"<img_00001>": 32001,
|
| 9 |
+
"<img_00002>": 32002,
|
| 10 |
+
"<img_00003>": 32003,
|
| 11 |
+
"<img_00004>": 32004,
|
| 12 |
+
"<img_00005>": 32005,
|
| 13 |
+
"<img_00006>": 32006,
|
| 14 |
+
"<img_00007>": 32007,
|
| 15 |
+
"<img_00008>": 32008,
|
| 16 |
+
"<img_00009>": 32009,
|
| 17 |
+
"<img_00010>": 32010,
|
| 18 |
+
"<img_00011>": 32011,
|
| 19 |
+
"<img_00012>": 32012,
|
| 20 |
+
"<img_00013>": 32013,
|
| 21 |
+
"<img_00014>": 32014,
|
| 22 |
+
"<img_00015>": 32015,
|
| 23 |
+
"<img_00016>": 32016,
|
| 24 |
+
"<img_00017>": 32017,
|
| 25 |
+
"<img_00018>": 32018,
|
| 26 |
+
"<img_00019>": 32019,
|
| 27 |
+
"<img_00020>": 32020,
|
| 28 |
+
"<img_00021>": 32021,
|
| 29 |
+
"<img_00022>": 32022,
|
| 30 |
+
"<img_00023>": 32023,
|
| 31 |
+
"<img_00024>": 32024,
|
| 32 |
+
"<img_00025>": 32025,
|
| 33 |
+
"<img_00026>": 32026,
|
| 34 |
+
"<img_00027>": 32027,
|
| 35 |
+
"<img_00028>": 32028,
|
| 36 |
+
"<img_00029>": 32029,
|
| 37 |
+
"<img_00030>": 32030,
|
| 38 |
+
"<img_00031>": 32031,
|
| 39 |
+
"<img_00032>": 32032,
|
| 40 |
+
"<img_00033>": 32033,
|
| 41 |
+
"<img_00034>": 32034,
|
| 42 |
+
"<img_00035>": 32035,
|
| 43 |
+
"<img_00036>": 32036,
|
| 44 |
+
"<img_00037>": 32037,
|
| 45 |
+
"<img_00038>": 32038,
|
| 46 |
+
"<img_00039>": 32039,
|
| 47 |
+
"<img_00040>": 32040,
|
| 48 |
+
"<img_00041>": 32041,
|
| 49 |
+
"<img_00042>": 32042,
|
| 50 |
+
"<img_00043>": 32043,
|
| 51 |
+
"<img_00044>": 32044,
|
| 52 |
+
"<img_00045>": 32045,
|
| 53 |
+
"<img_00046>": 32046,
|
| 54 |
+
"<img_00047>": 32047,
|
| 55 |
+
"<img_00048>": 32048,
|
| 56 |
+
"<img_00049>": 32049,
|
| 57 |
+
"<img_00050>": 32050,
|
| 58 |
+
"<img_00051>": 32051,
|
| 59 |
+
"<img_00052>": 32052,
|
| 60 |
+
"<img_00053>": 32053,
|
| 61 |
+
"<img_00054>": 32054,
|
| 62 |
+
"<img_00055>": 32055,
|
| 63 |
+
"<img_00056>": 32056,
|
| 64 |
+
"<img_00057>": 32057,
|
| 65 |
+
"<img_00058>": 32058,
|
| 66 |
+
"<img_00059>": 32059,
|
| 67 |
+
"<img_00060>": 32060,
|
| 68 |
+
"<img_00061>": 32061,
|
| 69 |
+
"<img_00062>": 32062,
|
| 70 |
+
"<img_00063>": 32063,
|
| 71 |
+
"<img_00064>": 32064,
|
| 72 |
+
"<img_00065>": 32065,
|
| 73 |
+
"<img_00066>": 32066,
|
| 74 |
+
"<img_00067>": 32067,
|
| 75 |
+
"<img_00068>": 32068,
|
| 76 |
+
"<img_00069>": 32069,
|
| 77 |
+
"<img_00070>": 32070,
|
| 78 |
+
"<img_00071>": 32071,
|
| 79 |
+
"<img_00072>": 32072,
|
| 80 |
+
"<img_00073>": 32073,
|
| 81 |
+
"<img_00074>": 32074,
|
| 82 |
+
"<img_00075>": 32075,
|
| 83 |
+
"<img_00076>": 32076,
|
| 84 |
+
"<img_00077>": 32077,
|
| 85 |
+
"<img_00078>": 32078,
|
| 86 |
+
"<img_00079>": 32079,
|
| 87 |
+
"<img_00080>": 32080,
|
| 88 |
+
"<img_00081>": 32081,
|
| 89 |
+
"<img_00082>": 32082,
|
| 90 |
+
"<img_00083>": 32083,
|
| 91 |
+
"<img_00084>": 32084,
|
| 92 |
+
"<img_00085>": 32085,
|
| 93 |
+
"<img_00086>": 32086,
|
| 94 |
+
"<img_00087>": 32087,
|
| 95 |
+
"<img_00088>": 32088,
|
| 96 |
+
"<img_00089>": 32089,
|
| 97 |
+
"<img_00090>": 32090,
|
| 98 |
+
"<img_00091>": 32091,
|
| 99 |
+
"<img_00092>": 32092,
|
| 100 |
+
"<img_00093>": 32093,
|
| 101 |
+
"<img_00094>": 32094,
|
| 102 |
+
"<img_00095>": 32095,
|
| 103 |
+
"<img_00096>": 32096,
|
| 104 |
+
"<img_00097>": 32097,
|
| 105 |
+
"<img_00098>": 32098,
|
| 106 |
+
"<img_00099>": 32099,
|
| 107 |
+
"<loc-0>": 32104,
|
| 108 |
+
"<loc-100>": 32204,
|
| 109 |
+
"<loc-101>": 32205,
|
| 110 |
+
"<loc-102>": 32206,
|
| 111 |
+
"<loc-103>": 32207,
|
| 112 |
+
"<loc-104>": 32208,
|
| 113 |
+
"<loc-105>": 32209,
|
| 114 |
+
"<loc-106>": 32210,
|
| 115 |
+
"<loc-107>": 32211,
|
| 116 |
+
"<loc-108>": 32212,
|
| 117 |
+
"<loc-109>": 32213,
|
| 118 |
+
"<loc-10>": 32114,
|
| 119 |
+
"<loc-110>": 32214,
|
| 120 |
+
"<loc-111>": 32215,
|
| 121 |
+
"<loc-112>": 32216,
|
| 122 |
+
"<loc-113>": 32217,
|
| 123 |
+
"<loc-114>": 32218,
|
| 124 |
+
"<loc-115>": 32219,
|
| 125 |
+
"<loc-116>": 32220,
|
| 126 |
+
"<loc-117>": 32221,
|
| 127 |
+
"<loc-118>": 32222,
|
| 128 |
+
"<loc-119>": 32223,
|
| 129 |
+
"<loc-11>": 32115,
|
| 130 |
+
"<loc-120>": 32224,
|
| 131 |
+
"<loc-121>": 32225,
|
| 132 |
+
"<loc-122>": 32226,
|
| 133 |
+
"<loc-123>": 32227,
|
| 134 |
+
"<loc-124>": 32228,
|
| 135 |
+
"<loc-125>": 32229,
|
| 136 |
+
"<loc-126>": 32230,
|
| 137 |
+
"<loc-127>": 32231,
|
| 138 |
+
"<loc-128>": 32232,
|
| 139 |
+
"<loc-129>": 32233,
|
| 140 |
+
"<loc-12>": 32116,
|
| 141 |
+
"<loc-130>": 32234,
|
| 142 |
+
"<loc-131>": 32235,
|
| 143 |
+
"<loc-132>": 32236,
|
| 144 |
+
"<loc-133>": 32237,
|
| 145 |
+
"<loc-134>": 32238,
|
| 146 |
+
"<loc-135>": 32239,
|
| 147 |
+
"<loc-136>": 32240,
|
| 148 |
+
"<loc-137>": 32241,
|
| 149 |
+
"<loc-138>": 32242,
|
| 150 |
+
"<loc-139>": 32243,
|
| 151 |
+
"<loc-13>": 32117,
|
| 152 |
+
"<loc-140>": 32244,
|
| 153 |
+
"<loc-141>": 32245,
|
| 154 |
+
"<loc-142>": 32246,
|
| 155 |
+
"<loc-143>": 32247,
|
| 156 |
+
"<loc-144>": 32248,
|
| 157 |
+
"<loc-145>": 32249,
|
| 158 |
+
"<loc-146>": 32250,
|
| 159 |
+
"<loc-147>": 32251,
|
| 160 |
+
"<loc-148>": 32252,
|
| 161 |
+
"<loc-149>": 32253,
|
| 162 |
+
"<loc-14>": 32118,
|
| 163 |
+
"<loc-150>": 32254,
|
| 164 |
+
"<loc-151>": 32255,
|
| 165 |
+
"<loc-152>": 32256,
|
| 166 |
+
"<loc-153>": 32257,
|
| 167 |
+
"<loc-154>": 32258,
|
| 168 |
+
"<loc-155>": 32259,
|
| 169 |
+
"<loc-156>": 32260,
|
| 170 |
+
"<loc-157>": 32261,
|
| 171 |
+
"<loc-158>": 32262,
|
| 172 |
+
"<loc-159>": 32263,
|
| 173 |
+
"<loc-15>": 32119,
|
| 174 |
+
"<loc-160>": 32264,
|
| 175 |
+
"<loc-161>": 32265,
|
| 176 |
+
"<loc-162>": 32266,
|
| 177 |
+
"<loc-163>": 32267,
|
| 178 |
+
"<loc-164>": 32268,
|
| 179 |
+
"<loc-165>": 32269,
|
| 180 |
+
"<loc-166>": 32270,
|
| 181 |
+
"<loc-167>": 32271,
|
| 182 |
+
"<loc-168>": 32272,
|
| 183 |
+
"<loc-169>": 32273,
|
| 184 |
+
"<loc-16>": 32120,
|
| 185 |
+
"<loc-170>": 32274,
|
| 186 |
+
"<loc-171>": 32275,
|
| 187 |
+
"<loc-172>": 32276,
|
| 188 |
+
"<loc-173>": 32277,
|
| 189 |
+
"<loc-174>": 32278,
|
| 190 |
+
"<loc-175>": 32279,
|
| 191 |
+
"<loc-176>": 32280,
|
| 192 |
+
"<loc-177>": 32281,
|
| 193 |
+
"<loc-178>": 32282,
|
| 194 |
+
"<loc-179>": 32283,
|
| 195 |
+
"<loc-17>": 32121,
|
| 196 |
+
"<loc-180>": 32284,
|
| 197 |
+
"<loc-181>": 32285,
|
| 198 |
+
"<loc-182>": 32286,
|
| 199 |
+
"<loc-183>": 32287,
|
| 200 |
+
"<loc-184>": 32288,
|
| 201 |
+
"<loc-185>": 32289,
|
| 202 |
+
"<loc-186>": 32290,
|
| 203 |
+
"<loc-187>": 32291,
|
| 204 |
+
"<loc-188>": 32292,
|
| 205 |
+
"<loc-189>": 32293,
|
| 206 |
+
"<loc-18>": 32122,
|
| 207 |
+
"<loc-190>": 32294,
|
| 208 |
+
"<loc-191>": 32295,
|
| 209 |
+
"<loc-192>": 32296,
|
| 210 |
+
"<loc-193>": 32297,
|
| 211 |
+
"<loc-194>": 32298,
|
| 212 |
+
"<loc-195>": 32299,
|
| 213 |
+
"<loc-196>": 32300,
|
| 214 |
+
"<loc-197>": 32301,
|
| 215 |
+
"<loc-198>": 32302,
|
| 216 |
+
"<loc-199>": 32303,
|
| 217 |
+
"<loc-19>": 32123,
|
| 218 |
+
"<loc-1>": 32105,
|
| 219 |
+
"<loc-200>": 32304,
|
| 220 |
+
"<loc-201>": 32305,
|
| 221 |
+
"<loc-202>": 32306,
|
| 222 |
+
"<loc-203>": 32307,
|
| 223 |
+
"<loc-204>": 32308,
|
| 224 |
+
"<loc-205>": 32309,
|
| 225 |
+
"<loc-206>": 32310,
|
| 226 |
+
"<loc-207>": 32311,
|
| 227 |
+
"<loc-208>": 32312,
|
| 228 |
+
"<loc-209>": 32313,
|
| 229 |
+
"<loc-20>": 32124,
|
| 230 |
+
"<loc-210>": 32314,
|
| 231 |
+
"<loc-211>": 32315,
|
| 232 |
+
"<loc-212>": 32316,
|
| 233 |
+
"<loc-213>": 32317,
|
| 234 |
+
"<loc-214>": 32318,
|
| 235 |
+
"<loc-215>": 32319,
|
| 236 |
+
"<loc-216>": 32320,
|
| 237 |
+
"<loc-217>": 32321,
|
| 238 |
+
"<loc-218>": 32322,
|
| 239 |
+
"<loc-219>": 32323,
|
| 240 |
+
"<loc-21>": 32125,
|
| 241 |
+
"<loc-220>": 32324,
|
| 242 |
+
"<loc-221>": 32325,
|
| 243 |
+
"<loc-222>": 32326,
|
| 244 |
+
"<loc-223>": 32327,
|
| 245 |
+
"<loc-22>": 32126,
|
| 246 |
+
"<loc-23>": 32127,
|
| 247 |
+
"<loc-24>": 32128,
|
| 248 |
+
"<loc-25>": 32129,
|
| 249 |
+
"<loc-26>": 32130,
|
| 250 |
+
"<loc-27>": 32131,
|
| 251 |
+
"<loc-28>": 32132,
|
| 252 |
+
"<loc-29>": 32133,
|
| 253 |
+
"<loc-2>": 32106,
|
| 254 |
+
"<loc-30>": 32134,
|
| 255 |
+
"<loc-31>": 32135,
|
| 256 |
+
"<loc-32>": 32136,
|
| 257 |
+
"<loc-33>": 32137,
|
| 258 |
+
"<loc-34>": 32138,
|
| 259 |
+
"<loc-35>": 32139,
|
| 260 |
+
"<loc-36>": 32140,
|
| 261 |
+
"<loc-37>": 32141,
|
| 262 |
+
"<loc-38>": 32142,
|
| 263 |
+
"<loc-39>": 32143,
|
| 264 |
+
"<loc-3>": 32107,
|
| 265 |
+
"<loc-40>": 32144,
|
| 266 |
+
"<loc-41>": 32145,
|
| 267 |
+
"<loc-42>": 32146,
|
| 268 |
+
"<loc-43>": 32147,
|
| 269 |
+
"<loc-44>": 32148,
|
| 270 |
+
"<loc-45>": 32149,
|
| 271 |
+
"<loc-46>": 32150,
|
| 272 |
+
"<loc-47>": 32151,
|
| 273 |
+
"<loc-48>": 32152,
|
| 274 |
+
"<loc-49>": 32153,
|
| 275 |
+
"<loc-4>": 32108,
|
| 276 |
+
"<loc-50>": 32154,
|
| 277 |
+
"<loc-51>": 32155,
|
| 278 |
+
"<loc-52>": 32156,
|
| 279 |
+
"<loc-53>": 32157,
|
| 280 |
+
"<loc-54>": 32158,
|
| 281 |
+
"<loc-55>": 32159,
|
| 282 |
+
"<loc-56>": 32160,
|
| 283 |
+
"<loc-57>": 32161,
|
| 284 |
+
"<loc-58>": 32162,
|
| 285 |
+
"<loc-59>": 32163,
|
| 286 |
+
"<loc-5>": 32109,
|
| 287 |
+
"<loc-60>": 32164,
|
| 288 |
+
"<loc-61>": 32165,
|
| 289 |
+
"<loc-62>": 32166,
|
| 290 |
+
"<loc-63>": 32167,
|
| 291 |
+
"<loc-64>": 32168,
|
| 292 |
+
"<loc-65>": 32169,
|
| 293 |
+
"<loc-66>": 32170,
|
| 294 |
+
"<loc-67>": 32171,
|
| 295 |
+
"<loc-68>": 32172,
|
| 296 |
+
"<loc-69>": 32173,
|
| 297 |
+
"<loc-6>": 32110,
|
| 298 |
+
"<loc-70>": 32174,
|
| 299 |
+
"<loc-71>": 32175,
|
| 300 |
+
"<loc-72>": 32176,
|
| 301 |
+
"<loc-73>": 32177,
|
| 302 |
+
"<loc-74>": 32178,
|
| 303 |
+
"<loc-75>": 32179,
|
| 304 |
+
"<loc-76>": 32180,
|
| 305 |
+
"<loc-77>": 32181,
|
| 306 |
+
"<loc-78>": 32182,
|
| 307 |
+
"<loc-79>": 32183,
|
| 308 |
+
"<loc-7>": 32111,
|
| 309 |
+
"<loc-80>": 32184,
|
| 310 |
+
"<loc-81>": 32185,
|
| 311 |
+
"<loc-82>": 32186,
|
| 312 |
+
"<loc-83>": 32187,
|
| 313 |
+
"<loc-84>": 32188,
|
| 314 |
+
"<loc-85>": 32189,
|
| 315 |
+
"<loc-86>": 32190,
|
| 316 |
+
"<loc-87>": 32191,
|
| 317 |
+
"<loc-88>": 32192,
|
| 318 |
+
"<loc-89>": 32193,
|
| 319 |
+
"<loc-8>": 32112,
|
| 320 |
+
"<loc-90>": 32194,
|
| 321 |
+
"<loc-91>": 32195,
|
| 322 |
+
"<loc-92>": 32196,
|
| 323 |
+
"<loc-93>": 32197,
|
| 324 |
+
"<loc-94>": 32198,
|
| 325 |
+
"<loc-95>": 32199,
|
| 326 |
+
"<loc-96>": 32200,
|
| 327 |
+
"<loc-97>": 32201,
|
| 328 |
+
"<loc-98>": 32202,
|
| 329 |
+
"<loc-99>": 32203,
|
| 330 |
+
"<loc-9>": 32113,
|
| 331 |
+
"<patch>": 32102
|
| 332 |
+
}
|
cvlm_llama2_tokenizer_100img_and_224loc_addpatch/special_tokens_map.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "</s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": "<unk>",
|
| 17 |
+
"unk_token": {
|
| 18 |
+
"content": "<unk>",
|
| 19 |
+
"lstrip": false,
|
| 20 |
+
"normalized": false,
|
| 21 |
+
"rstrip": false,
|
| 22 |
+
"single_word": false
|
| 23 |
+
}
|
| 24 |
+
}
|
cvlm_llama2_tokenizer_100img_and_224loc_addpatch/tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
|
| 3 |
+
size 499723
|
cvlm_llama2_tokenizer_100img_and_224loc_addpatch/tokenizer_config.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": true,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"bos_token": {
|
| 5 |
+
"__type": "AddedToken",
|
| 6 |
+
"content": "<s>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false
|
| 11 |
+
},
|
| 12 |
+
"clean_up_tokenization_spaces": false,
|
| 13 |
+
"eos_token": {
|
| 14 |
+
"__type": "AddedToken",
|
| 15 |
+
"content": "</s>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false
|
| 20 |
+
},
|
| 21 |
+
"legacy": false,
|
| 22 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 23 |
+
"pad_token": null,
|
| 24 |
+
"sp_model_kwargs": {},
|
| 25 |
+
"tokenizer_class": "LlamaTokenizer",
|
| 26 |
+
"unk_token": {
|
| 27 |
+
"__type": "AddedToken",
|
| 28 |
+
"content": "<unk>",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"rstrip": false,
|
| 32 |
+
"single_word": false
|
| 33 |
+
}
|
| 34 |
+
}
|