data/lang_bpe_500/tokens.txt

<blk> 0
<sos/eos> 1
<unk> 2
S 3
▁THE 4
▁A 5
T 6
▁AND 7
ED 8
▁OF 9
▁TO 10
E 11
D 12
N 13
ING 14
▁IN 15
Y 16
M 17
C 18
▁I 19
A 20
P 21
▁HE 22
R 23
O 24
L 25
RE 26
I 27
U 28
ER 29
▁IT 30
LY 31
▁THAT 32
▁WAS 33
▁ 34
▁S 35
AR 36
▁BE 37
F 38
▁C 39
IN 40
B 41
▁FOR 42
OR 43
LE 44
' 45
▁HIS 46
▁YOU 47
AL 48
▁RE 49
V 50
▁B 51
G 52
RI 53
▁E 54
▁WITH 55
▁T 56
▁AS 57
LL 58
▁P 59
▁HER 60
ST 61
▁HAD 62
▁SO 63
▁F 64
W 65
CE 66
▁IS 67
ND 68
▁NOT 69
TH 70
▁BUT 71
EN 72
▁SHE 73
▁ON 74
VE 75
ON 76
SE 77
▁DE 78
UR 79
▁G 80
CH 81
K 82
TER 83
▁AT 84
IT 85
▁ME 86
RO 87
NE 88
RA 89
ES 90
IL 91
NG 92
IC 93
▁NO 94
▁HIM 95
ENT 96
IR 97
▁WE 98
H 99
▁DO 100
▁ALL 101
▁HAVE 102
LO 103
▁BY 104
▁MY 105
▁MO 106
▁THIS 107
LA 108
▁ST 109
▁WHICH 110
▁CON 111
▁THEY 112
CK 113
TE 114
▁SAID 115
▁FROM 116
▁GO 117
▁WHO 118
▁TH 119
▁OR 120
▁D 121
▁W 122
VER 123
LI 124
▁SE 125
▁ONE 126
▁CA 127
▁AN 128
▁LA 129
▁WERE 130
EL 131
▁HA 132
▁MAN 133
▁FA 134
▁EX 135
AD 136
▁SU 137
RY 138
▁MI 139
AT 140
▁BO 141
▁WHEN 142
AN 143
THER 144
PP 145
ATION 146
▁FI 147
▁WOULD 148
▁PRO 149
OW 150
ET 151
▁O 152
▁THERE 153
▁HO 154
ION 155
▁WHAT 156
▁FE 157
▁PA 158
US 159
MENT 160
▁MA 161
UT 162
▁OUT 163
▁THEIR 164
▁IF 165
▁LI 166
▁K 167
▁WILL 168
▁ARE 169
ID 170
▁RO 171
DE 172
TION 173
▁WA 174
PE 175
▁UP 176
▁SP 177
▁PO 178
IGHT 179
▁UN 180
RU 181
▁LO 182
AS 183
OL 184
▁LE 185
▁BEEN 186
▁SH 187
▁RA 188
▁SEE 189
KE 190
UL 191
TED 192
▁SA 193
UN 194
UND 195
ANT 196
▁NE 197
IS 198
▁THEM 199
CI 200
GE 201
▁COULD 202
▁DIS 203
OM 204
ISH 205
HE 206
EST 207
▁SOME 208
ENCE 209
ITY 210
IVE 211
▁US 212
▁MORE 213
▁EN 214
ARD 215
ATE 216
▁YOUR 217
▁INTO 218
▁KNOW 219
▁CO 220
ANCE 221
▁TIME 222
▁WI 223
▁YE 224
AGE 225
▁NOW 226
TI 227
FF 228
ABLE 229
▁VERY 230
▁LIKE 231
AM 232
HI 233
Z 234
▁OTHER 235
▁THAN 236
▁LITTLE 237
▁DID 238
▁LOOK 239
TY 240
ERS 241
▁CAN 242
▁CHA 243
▁AR 244
X 245
FUL 246
UGH 247
▁BA 248
▁DAY 249
▁ABOUT 250
TEN 251
IM 252
▁ANY 253
▁PRE 254
▁OVER 255
IES 256
NESS 257
ME 258
BLE 259
▁M 260
ROW 261
▁HAS 262
▁GREAT 263
▁VI 264
TA 265
▁AFTER 266
PER 267
▁AGAIN 268
HO 269
SH 270
▁UPON 271
▁DI 272
▁HAND 273
▁COM 274
IST 275
TURE 276
▁STA 277
▁THEN 278
▁SHOULD 279
▁GA 280
OUS 281
OUR 282
▁WELL 283
▁ONLY 284
MAN 285
▁GOOD 286
▁TWO 287
▁MAR 288
▁SAY 289
▁HU 290
TING 291
▁OUR 292
RESS 293
▁DOWN 294
IOUS 295
▁BEFORE 296
▁DA 297
▁NA 298
QUI 299
▁MADE 300
▁EVERY 301
▁OLD 302
▁EVEN 303
IG 304
▁COME 305
▁GRA 306
▁RI 307
▁LONG 308
OT 309
SIDE 310
WARD 311
▁FO 312
▁WHERE 313
MO 314
LESS 315
▁SC 316
▁MUST 317
▁NEVER 318
▁HOW 319
▁CAME 320
▁SUCH 321
▁RU 322
▁TAKE 323
▁WO 324
▁CAR 325
UM 326
AK 327
▁THINK 328
▁MUCH 329
▁MISTER 330
▁MAY 331
▁JO 332
▁WAY 333
▁COMP 334
▁THOUGHT 335
▁STO 336
▁MEN 337
▁BACK 338
▁DON 339
J 340
▁LET 341
▁TRA 342
▁FIRST 343
▁JUST 344
▁VA 345
▁OWN 346
▁PLA 347
▁MAKE 348
ATED 349
▁HIMSELF 350
▁WENT 351
▁PI 352
GG 353
RING 354
▁DU 355
▁MIGHT 356
▁PART 357
▁GIVE 358
▁IMP 359
▁BU 360
▁PER 361
▁PLACE 362
▁HOUSE 363
▁THROUGH 364
IAN 365
▁SW 366
▁UNDER 367
QUE 368
▁AWAY 369
▁LOVE 370
QUA 371
▁LIFE 372
▁GET 373
▁WITHOUT 374
▁PASS 375
▁TURN 376
IGN 377
▁HEAD 378
▁MOST 379
▁THOSE 380
▁SHALL 381
▁EYES 382
▁COL 383
▁STILL 384
▁NIGHT 385
▁NOTHING 386
ITION 387
HA 388
▁TELL 389
▁WORK 390
▁LAST 391
▁NEW 392
▁FACE 393
▁HI 394
▁WORD 395
▁FOUND 396
▁COUNT 397
▁OB 398
▁WHILE 399
▁SHA 400
▁MEAN 401
▁SAW 402
▁PEOPLE 403
▁FRIEND 404
▁THREE 405
▁ROOM 406
▁SAME 407
▁THOUGH 408
▁RIGHT 409
▁CHILD 410
▁FATHER 411
▁ANOTHER 412
▁HEART 413
▁WANT 414
▁TOOK 415
OOK 416
▁LIGHT 417
▁MISSUS 418
▁OPEN 419
▁JU 420
▁ASKED 421
PORT 422
▁LEFT 423
▁JA 424
▁WORLD 425
▁HOME 426
▁WHY 427
▁ALWAYS 428
▁ANSWER 429
▁SEEMED 430
▁SOMETHING 431
▁GIRL 432
▁BECAUSE 433
▁NAME 434
▁TOLD 435
▁NI 436
▁HIGH 437
IZE 438
▁WOMAN 439
▁FOLLOW 440
▁RETURN 441
▁KNEW 442
▁EACH 443
▁KIND 444
▁JE 445
▁ACT 446
▁LU 447
▁CERTAIN 448
▁YEARS 449
▁QUITE 450
▁APPEAR 451
▁BETTER 452
▁HALF 453
▁PRESENT 454
▁PRINCE 455
SHIP 456
▁ALSO 457
▁BEGAN 458
▁HAVING 459
▁ENOUGH 460
▁PERSON 461
▁LADY 462
▁WHITE 463
▁COURSE 464
▁VOICE 465
▁SPEAK 466
▁POWER 467
▁MORNING 468
▁BETWEEN 469
▁AMONG 470
▁KEEP 471
▁WALK 472
▁MATTER 473
▁TEA 474
▁BELIEVE 475
▁SMALL 476
▁TALK 477
▁FELT 478
▁HORSE 479
▁MYSELF 480
▁SIX 481
▁HOWEVER 482
▁FULL 483
▁HERSELF 484
▁POINT 485
▁STOOD 486
▁HUNDRED 487
▁ALMOST 488
▁SINCE 489
▁LARGE 490
▁LEAVE 491
▁PERHAPS 492
▁DARK 493
▁SUDDEN 494
▁REPLIED 495
▁ANYTHING 496
▁WONDER 497
▁UNTIL 498
Q 499
#0 500
#1 501