added tokens and words

README.md

@@ -111,6 +111,6 @@ export CUDA_VISIBLE_DEVICES="0,1,2,3"
 ```
 
 The tensorboard training log can be found at
-https://tensorboard.dev/experiment/AKXbJha0S9aXyfmuvG4h5A/#scalars
+https://tensorboard.dev/experiment/3d4bYmbJTGiWQQaW88CVEQ/#scalars
 
 The decoding commands are same as above.

data/lang_bpe_500/tokens.txt

@@ -0,0 +1,502 @@
+<blk> 0
+<sos/eos> 1
+<unk> 2
+s 3
+t 4
+▁the 5
+▁a 6
+▁to 7
+n 8
+▁and 9
+e 10
+ing 11
+▁of 12
+▁in 13
+d 14
+▁ 15
+y 16
+ed 17
+a 18
+r 19
+o 20
+p 21
+m 22
+▁i 23
+g 24
+c 25
+re 26
+▁that 27
+▁it 28
+er 29
+b 30
+al 31
+▁is 32
+▁s 33
+u 34
+▁re 35
+' 36
+i 37
+▁c 38
+or 39
+l 40
+ly 41
+▁be 42
+▁for 43
+ar 44
+w 45
+in 46
+k 47
+le 48
+▁f 49
+▁we 50
+f 51
+▁he 52
+▁e 53
+▁you 54
+ll 55
+▁w 56
+▁on 57
+ce 58
+it 59
+ve 60
+▁are 61
+ri 62
+▁with 63
+▁was 64
+▁t 65
+▁b 66
+ent 67
+▁de 68
+ic 69
+v 70
+▁so 71
+▁have 72
+es 73
+ur 74
+ch 75
+th 76
+▁but 77
+▁st 78
+nd 79
+en 80
+on 81
+ra 82
+h 83
+te 84
+ter 85
+▁this 86
+▁one 87
+▁they 88
+▁not 89
+ro 90
+ation 91
+▁as 92
+▁me 93
+an 94
+▁do 95
+▁or 96
+li 97
+se 98
+il 99
+▁can 100
+▁mo 101
+▁pa 102
+▁con 103
+▁ca 104
+ck 105
+▁ma 106
+▁su 107
+la 108
+▁an 109
+ate 110
+▁at 111
+lo 112
+▁co 113
+ne 114
+el 115
+ng 116
+▁from 117
+at 118
+▁g 119
+▁le 120
+▁lo 121
+▁said 122
+ver 123
+ad 124
+me 125
+▁po 126
+▁us 127
+▁se 128
+▁all 129
+▁what 130
+ers 131
+▁no 132
+as 133
+▁te 134
+ment 135
+ect 136
+ies 137
+▁there 138
+▁fa 139
+ry 140
+▁ha 141
+▁o 142
+▁go 143
+▁bo 144
+▁about 145
+▁by 146
+▁my 147
+▁more 148
+▁who 149
+ir 150
+▁pro 151
+ci 152
+▁out 153
+▁will 154
+▁people 155
+ge 156
+▁their 157
+▁if 158
+▁up 159
+▁his 160
+▁mi 161
+▁la 162
+ity 163
+▁' 164
+▁un 165
+z 166
+▁when 167
+▁ho 168
+▁like 169
+▁ba 170
+▁th 171
+pe 172
+▁has 173
+ion 174
+vi 175
+▁two 176
+age 177
+ure 178
+de 179
+▁she 180
+x 181
+▁some 182
+▁time 183
+lu 184
+ul 185
+ive 186
+ist 187
+et 188
+un 189
+ol 190
+▁en 191
+▁would 192
+▁pre 193
+▁get 194
+▁ra 195
+▁our 196
+▁had 197
+able 198
+act 199
+ta 200
+▁just 201
+▁your 202
+co 203
+▁li 204
+▁thousand 205
+▁see 206
+▁were 207
+tion 208
+ma 209
+im 210
+▁sp 211
+▁hundred 212
+mp 213
+▁how 214
+▁ro 215
+▁ex 216
+us 217
+ough 218
+ru 219
+▁vi 220
+▁comp 221
+mo 222
+j 223
+▁dr 224
+is 225
+▁her 226
+▁which 227
+▁work 228
+▁than 229
+est 230
+▁other 231
+ting 232
+ke 233
+▁dis 234
+om 235
+▁say 236
+ut 237
+ld 238
+▁know 239
+▁new 240
+ti 241
+▁been 242
+ard 243
+ther 244
+he 245
+▁pu 246
+ine 247
+qu 248
+▁fi 249
+▁fe 250
+ical 251
+▁di 252
+▁them 253
+ow 254
+ight 255
+one 256
+▁every 257
+di 258
+id 259
+▁bi 260
+ach 261
+▁da 262
+▁make 263
+▁any 264
+▁sa 265
+▁com 266
+▁think 267
+▁even 268
+▁want 269
+▁into 270
+▁now 271
+ance 272
+ot 273
+ff 274
+▁way 275
+▁over 276
+pp 277
+▁mu 278
+▁nine 279
+ence 280
+▁exp 281
+▁because 282
+▁ga 283
+les 284
+▁could 285
+▁na 286
+ant 287
+▁ki 288
+ig 289
+ction 290
+▁very 291
+und 292
+side 293
+▁sta 294
+ho 295
+ish 296
+▁don 297
+▁per 298
+tic 299
+ated 300
+ial 301
+▁ne 302
+▁look 303
+▁tra 304
+▁k 305
+▁pi 306
+ca 307
+▁most 308
+▁need 309
+▁bu 310
+if 311
+um 312
+ving 313
+▁these 314
+▁only 315
+▁take 316
+▁part 317
+▁off 318
+ful 319
+▁back 320
+▁jo 321
+mb 322
+▁three 323
+▁years 324
+▁day 325
+▁car 326
+▁also 327
+▁sh 328
+▁going 329
+▁world 330
+▁where 331
+cu 332
+▁ri 333
+qui 334
+▁did 335
+▁app 336
+▁sc 337
+port 338
+der 339
+▁ta 340
+▁five 341
+▁first 342
+▁much 343
+▁many 344
+▁good 345
+▁ru 346
+end 347
+▁six 348
+▁him 349
+ous 350
+▁pri 351
+▁ph 352
+▁may 353
+▁play 354
+ition 355
+▁those 356
+▁life 357
+▁man 358
+ian 359
+▁wi 360
+ha 361
+ph 362
+▁four 363
+▁cha 364
+▁eight 365
+▁after 366
+▁come 367
+▁really 368
+▁va 369
+▁year 370
+▁count 371
+▁well 372
+▁long 373
+▁through 374
+iv 375
+▁down 376
+▁seven 377
+form 378
+ize 379
+▁right 380
+▁should 381
+ual 382
+▁chi 383
+▁show 384
+▁gu 385
+▁imp 386
+ious 387
+▁twenty 388
+ssion 389
+▁point 390
+▁help 391
+▁hu 392
+▁does 393
+▁gra 394
+▁pl 395
+ible 396
+▁things 397
+▁high 398
+▁something 399
+▁great 400
+▁own 401
+▁start 402
+▁hi 403
+fi 404
+▁still 405
+man 406
+▁give 407
+▁change 408
+▁feel 409
+▁sha 410
+▁live 411
+▁around 412
+▁mean 413
+▁plan 414
+▁again 415
+▁place 416
+▁before 417
+▁old 418
+▁find 419
+▁ye 420
+▁made 421
+▁home 422
+▁same 423
+▁never 424
+▁add 425
+▁little 426
+ative 427
+▁gene 428
+▁last 429
+▁different 430
+▁america 431
+▁person 432
+▁percent 433
+▁happen 434
+▁problem 435
+▁might 436
+▁under 437
+▁better 438
+▁while 439
+▁children 440
+▁another 441
+▁why 442
+▁turn 443
+▁spec 444
+▁hard 445
+▁school 446
+▁system 447
+ship 448
+▁always 449
+▁found 450
+▁talk 451
+▁ja 452
+▁power 453
+▁thirty 454
+▁human 455
+▁week 456
+▁idea 457
+▁develop 458
+▁keep 459
+▁between 460
+▁trans 461
+▁question 462
+▁grow 463
+▁small 464
+▁money 465
+▁important 466
+▁women 467
+▁next 468
+▁ju 469
+▁fifty 470
+▁become 471
+▁friend 472
+▁family 473
+▁without 474
+▁build 475
+▁number 476
+▁today 477
+▁large 478
+▁learn 479
+▁health 480
+▁believe 481
+▁house 482
+▁business 483
+▁open 484
+▁interest 485
+▁sign 486
+▁research 487
+▁government 488
+▁experience 489
+▁market 490
+▁course 491
+▁light 492
+▁product 493
+▁second 494
+▁together 495
+▁close 496
+▁understand 497
+▁group 498
+q 499
+#0 500
+#1 501