data/lang_bpe_500/tokens.txt

<blk> 0
<sos/eos> 1
<unk> 2
S 3
▁THE 4
T 5
▁A 6
E 7
▁AND 8
ED 9
D 10
▁TO 11
▁OF 12
N 13
ING 14
C 15
▁IN 16
Y 17
O 18
M 19
▁I 20
A 21
I 22
▁S 23
P 24
▁HE 25
R 26
▁ 27
▁IT 28
LY 29
▁WAS 30
▁F 31
▁THAT 32
AR 33
ER 34
L 35
B 36
' 37
▁BE 38
▁YOU 39
RE 40
▁FOR 41
▁RE 42
▁HIS 43
LE 44
OR 45
AL 46
U 47
IN 48
G 49
V 50
▁P 51
ON 52
▁WITH 53
▁C 54
W 55
▁E 56
IT 57
▁AS 58
K 59
▁HAD 60
EN 61
▁HER 62
▁W 63
ST 64
▁IS 65
LL 66
▁NOT 67
F 68
▁SHE 69
▁BUT 70
CE 71
▁SO 72
TH 73
TER 74
▁DE 75
▁ON 76
▁ME 77
H 78
RI 79
▁AT 80
CH 81
VE 82
▁ST 83
SE 84
IC 85
IL 86
ES 87
▁DO 88
▁B 89
▁G 90
ENT 91
IR 92
▁' 93
UR 94
ION 95
▁HIM 96
NG 97
▁HAVE 98
▁NO 99
▁WE 100
NE 101
▁ALL 102
▁MY 103
▁THIS 104
▁SAID 105
AN 106
RO 107
▁GO 108
LA 109
▁THEY 110
CK 111
▁BY 112
▁MO 113
AT 114
▁FROM 115
▁WHICH 116
VER 117
LO 118
▁TH 119
▁CON 120
▁ONE 121
RA 122
AD 123
▁WERE 124
▁WHO 125
▁CA 126
▁O 127
▁AN 128
▁OR 129
▁BO 130
▁THERE 131
LI 132
▁T 133
▁MAN 134
▁SH 135
▁SU 136
EL 137
▁EX 138
TE 139
▁WHAT 140
▁LA 141
ID 142
ET 143
▁TR 144
ATION 145
THER 146
▁WOULD 147
▁WHEN 148
OW 149
▁FA 150
▁PRO 151
RY 152
IS 153
▁ARE 154
▁WILL 155
▁OUT 156
▁HO 157
▁MA 158
PP 159
AS 160
▁IF 161
▁UP 162
▁BR 163
▁PA 164
RU 165
US 166
▁K 167
▁LI 168
▁SEE 169
▁HA 170
▁RO 171
UT 172
UN 173
IGHT 174
OL 175
▁PO 176
PE 177
▁DR 178
▁LO 179
▁THEIR 180
UL 181
▁BEEN 182
TED 183
ANT 184
▁UN 185
▁CH 186
▁RA 187
ATE 188
▁COULD 189
ISH 190
▁EN 191
IVE 192
▁LE 193
URE 194
▁WA 195
▁YOUR 196
AM 197
IG 198
▁MI 199
EST 200
MENT 201
▁THEM 202
▁DIS 203
▁SP 204
OO 205
▁KNOW 206
GE 207
▁CO 208
▁SOME 209
▁US 210
AIN 211
▁NE 212
ENCE 213
▁TIME 214
▁INTO 215
▁MR 216
▁MORE 217
▁NOW 218
▁DID 219
▁VERY 220
UND 221
AGE 222
ANCE 223
▁LOOK 224
▁CAN 225
▁LIKE 226
ME 227
ARD 228
ITY 229
KE 230
IE 231
TY 232
▁DI 233
UGH 234
LU 235
▁LITTLE 236
FF 237
ABLE 238
Z 239
▁ABOUT 240
▁OTHER 241
▁BA 242
FUL 243
IF 244
X 245
▁THAN 246
OUR 247
TEN 248
▁SA 249
PER 250
▁AR 251
▁HAS 252
▁PRE 253
▁CHA 254
▁ANY 255
▁OVER 256
▁DA 257
▁AGAIN 258
▁DAY 259
ERS 260
▁AFTER 261
▁HU 262
BLE 263
▁THEN 264
NESS 265
IM 266
▁HAND 267
HE 268
▁WELL 269
▁COM 270
IES 271
IST 272
▁MAR 273
▁GOOD 274
▁GREAT 275
AND 276
▁VI 277
SH 278
▁ONLY 279
▁SHOULD 280
▁PU 281
DER 282
ROW 283
▁UPON 284
▁COME 285
▁SAY 286
▁GA 287
▁DOWN 288
▁TWO 289
▁MUST 290
▁DON 291
OUS 292
IOUS 293
▁BEFORE 294
▁THINK 295
▁OUR 296
▁AM 297
UM 298
▁CAR 299
INE 300
CTI 301
▁MADE 302
▁RI 303
▁HOW 304
▁EVERY 305
▁GRA 306
▁OLD 307
FOR 308
▁NEVER 309
SIDE 310
▁EVEN 311
▁CAME 312
▁LONG 313
▁TAKE 314
WARD 315
EM 316
MAN 317
LESS 318
LED 319
▁OFF 320
▁QU 321
▁WHERE 322
AK 323
▁MUCH 324
▁BACK 325
▁SUCH 326
▁LET 327
RESS 328
UD 329
▁JO 330
▁PART 331
▁THOUGHT 332
▁WAY 333
▁JUST 334
J 335
▁DU 336
▁FIRST 337
▁MAY 338
▁COMP 339
▁STO 340
▁BU 341
▁MAKE 342
▁WENT 343
▁TA 344
GG 345
▁HIMSELF 346
▁VA 347
▁GET 348
▁OWN 349
▁GIVE 350
▁PLACE 351
▁MEN 352
▁IMP 353
▁HERE 354
▁MIGHT 355
ATED 356
▁AWAY 357
▁HOUSE 358
▁PER 359
▁LOVE 360
▁TURN 361
HA 362
TRA 363
▁TELL 364
LAND 365
▁MU 366
▁THROUGH 367
▁COL 368
▁UNDER 369
▁SHALL 370
▁LIFE 371
▁PASS 372
▁HEAD 373
▁EYES 374
QUI 375
▁NIGHT 376
▁WITHOUT 377
▁NOTHING 378
▁STILL 379
▁LAST 380
▁NA 381
▁FACE 382
▁WORK 383
▁MOST 384
▁NEW 385
▁TI 386
▁WANT 387
▁THOSE 388
▁MEAN 389
▁WORD 390
▁FOUND 391
▁PEOPLE 392
▁SAW 393
▁YES 394
▁THREE 395
▁ASKED 396
▁RIGHT 397
▁FRIEND 398
▁ROOM 399
▁YEAR 400
▁COUNT 401
▁HI 402
▁CHILD 403
▁WHILE 404
▁GU 405
▁ANOTHER 406
▁FATHER 407
▁TOOK 408
▁SEA 409
▁WHY 410
▁YET 411
▁HEART 412
▁OPEN 413
▁SAME 414
▁MOMENT 415
▁LIGHT 416
QUE 417
▁ANSWER 418
▁SOMETHING 419
▁THOUGH 420
PORT 421
▁LEFT 422
▁MIND 423
▁JA 424
▁SEEMED 425
▁FIND 426
▁JU 427
▁HOME 428
▁GIRL 429
▁ALWAYS 430
▁TOLD 431
▁QUITE 432
▁MISS 433
▁FEEL 434
▁FOLLOW 435
▁BECAUSE 436
▁WORLD 437
▁FELL 438
▁NI 439
▁RETURN 440
▁WOMAN 441
▁EACH 442
IZE 443
▁KNEW 444
▁NAME 445
▁HARD 446
▁KIND 447
▁JE 448
▁CERTAIN 449
▁HIGH 450
▁COURSE 451
▁BETTER 452
▁PRINCE 453
▁BEGAN 454
▁APPEAR 455
▁HALF 456
▁FEW 457
▁ENOUGH 458
▁SPEAK 459
▁PRESENT 460
▁WALK 461
SHIP 462
▁CLOSE 463
▁CRIED 464
▁KEEP 465
QUA 466
▁HELP 467
▁VOICE 468
▁LADY 469
▁TALK 470
▁ALSO 471
▁BELIEVE 472
▁MATTER 473
▁TEA 474
▁MORNING 475
▁REPLIED 476
▁FELT 477
▁PERSON 478
▁MYSELF 479
▁WHITE 480
▁HOWEVER 481
▁POWER 482
▁POINT 483
▁SIX 484
▁ANYTHING 485
▁PERHAPS 486
▁STOOD 487
▁FULL 488
▁SMALL 489
▁BETWEEN 490
▁HERSELF 491
▁ALMOST 492
▁HAVING 493
▁HORSE 494
▁UNTIL 495
▁NEXT 496
▁DARK 497
▁LEAVE 498
Q 499
#0 500
#1 501