Upload 10 files

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "data/models/Mixtral-8x7B-v0.1-prot",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "model_type": "mixtral",
+  "num_attention_heads": 8,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.2",
+  "use_cache": true,
+  "vocab_size": 1024
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.37.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e43be781fa4022ac89cf327e9d26fa5262da3ee82620837914218d77e63152b
+size 30480368

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b3fcd536a84053ef5e50c55cf33d490764fc847e4336d0e4903b704e487cdb9
+size 14308

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdab64b8110eed3c22ca8353445d05698be82fa26d1c6dd01fc983e50695283d
+size 1064

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer.json

@@ -0,0 +1,2091 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "vocab": {
+      "[UNK]": 0,
+      "[CLS]": 1,
+      "[SEP]": 2,
+      "[PAD]": 3,
+      "[MASK]": 4,
+      "A": 5,
+      "B": 6,
+      "C": 7,
+      "D": 8,
+      "E": 9,
+      "F": 10,
+      "G": 11,
+      "H": 12,
+      "I": 13,
+      "K": 14,
+      "L": 15,
+      "M": 16,
+      "N": 17,
+      "P": 18,
+      "Q": 19,
+      "R": 20,
+      "S": 21,
+      "T": 22,
+      "V": 23,
+      "W": 24,
+      "X": 25,
+      "Y": 26,
+      "Z": 27,
+      "AA": 28,
+      "SS": 29,
+      "TT": 30,
+      "GG": 31,
+      "LL": 32,
+      "AG": 33,
+      "LS": 34,
+      "TV": 35,
+      "AV": 36,
+      "AL": 37,
+      "AS": 38,
+      "DG": 39,
+      "TG": 40,
+      "EE": 41,
+      "TL": 42,
+      "TS": 43,
+      "DV": 44,
+      "AE": 45,
+      "DL": 46,
+      "SG": 47,
+      "AP": 48,
+      "RL": 49,
+      "VV": 50,
+      "SL": 51,
+      "TP": 52,
+      "TI": 53,
+      "EL": 54,
+      "SV": 55,
+      "AD": 56,
+      "NG": 57,
+      "AR": 58,
+      "EV": 59,
+      "QL": 60,
+      "TD": 61,
+      "AI": 62,
+      "NL": 63,
+      "EG": 64,
+      "PV": 65,
+      "KL": 66,
+      "PG": 67,
+      "EI": 68,
+      "NV": 69,
+      "DI": 70,
+      "PL": 71,
+      "DS": 72,
+      "KK": 73,
+      "RV": 74,
+      "NI": 75,
+      "AQ": 76,
+      "PS": 77,
+      "AT": 78,
+      "EK": 79,
+      "ES": 80,
+      "RG": 81,
+      "QQ": 82,
+      "NS": 83,
+      "DD": 84,
+      "AK": 85,
+      "RR": 86,
+      "VL": 87,
+      "AF": 88,
+      "TF": 89,
+      "RI": 90,
+      "PP": 91,
+      "QV": 92,
+      "TE": 93,
+      "AN": 94,
+      "SI": 95,
+      "KV": 96,
+      "QG": 97,
+      "KI": 98,
+      "RS": 99,
+      "TY": 100,
+      "DP": 101,
+      "NN": 102,
+      "KG": 103,
+      "DE": 104,
+      "QI": 105,
+      "FG": 106,
+      "VG": 107,
+      "YL": 108,
+      "FL": 109,
+      "TN": 110,
+      "PI": 111,
+      "KS": 112,
+      "QS": 113,
+      "PE": 114,
+      "AY": 115,
+      "HL": 116,
+      "RE": 117,
+      "FS": 118,
+      "FV": 119,
+      "TK": 120,
+      "GL": 121,
+      "VS": 122,
+      "TQ": 123,
+      "DN": 124,
+      "IL": 125,
+      "RP": 126,
+      "GS": 127,
+      "KE": 128,
+      "IS": 129,
+      "DF": 130,
+      "TR": 131,
+      "DR": 132,
+      "DY": 133,
+      "AH": 134,
+      "DK": 135,
+      "IG": 136,
+      "QE": 137,
+      "AM": 138,
+      "YG": 139,
+      "NE": 140,
+      "IV": 141,
+      "YS": 142,
+      "NP": 143,
+      "ER": 144,
+      "QP": 145,
+      "YV": 146,
+      "ML": 147,
+      "TA": 148,
+      "QR": 149,
+      "GV": 150,
+      "ND": 151,
+      "KP": 152,
+      "FE": 153,
+      "FI": 154,
+      "NK": 155,
+      "HS": 156,
+      "HG": 157,
+      "QK": 158,
+      "CL": 159,
+      "HV": 160,
+      "NY": 161,
+      "IE": 162,
+      "DQ": 163,
+      "WL": 164,
+      "RK": 165,
+      "CS": 166,
+      "NF": 167,
+      "RD": 168,
+      "EP": 169,
+      "RF": 170,
+      "AAL": 171,
+      "ED": 172,
+      "II": 173,
+      "TM": 174,
+      "TC": 175,
+      "NQ": 176,
+      "TH": 177,
+      "AGG": 178,
+      "FD": 179,
+      "AAG": 180,
+      "RQ": 181,
+      "AC": 182,
+      "PD": 183,
+      "VI": 184,
+      "EQ": 185,
+      "LG": 186,
+      "YI": 187,
+      "AW": 188,
+      "MS": 189,
+      "MV": 190,
+      "KD": 191,
+      "LV": 192,
+      "SSS": 193,
+      "NR": 194,
+      "CG": 195,
+      "HI": 196,
+      "PK": 197,
+      "TW": 198,
+      "RY": 199,
+      "EF": 200,
+      "EN": 201,
+      "ADG": 202,
+      "ALL": 203,
+      "PQ": 204,
+      "EY": 205,
+      "CV": 206,
+      "TAA": 207,
+      "PF": 208,
+      "XX": 209,
+      "TSS": 210,
+      "MG": 211,
+      "KQ": 212,
+      "ID": 213,
+      "PR": 214,
+      "TLS": 215,
+      "ASS": 216,
+      "QD": 217,
+      "RN": 218,
+      "WS": 219,
+      "RH": 220,
+      "FK": 221,
+      "VD": 222,
+      "VE": 223,
+      "KN": 224,
+      "TGG": 225,
+      "FF": 226,
+      "ASG": 227,
+      "QN": 228,
+      "ATT": 229,
+      "QY": 230,
+      "HP": 231,
+      "ATG": 232,
+      "ATV": 233,
+      "KY": 234,
+      "VP": 235,
+      "ALS": 236,
+      "QF": 237,
+      "IN": 238,
+      "TAG": 239,
+      "IK": 240,
+      "TAS": 241,
+      "SP": 242,
+      "YY": 243,
+      "FN": 244,
+      "LP": 245,
+      "IP": 246,
+      "YD": 247,
+      "EH": 248,
+      "TAV": 249,
+      "KR": 250,
+      "SD": 251,
+      "VR": 252,
+      "ATL": 253,
+      "GD": 254,
+      "EM": 255,
+      "TLL": 256,
+      "QH": 257,
+      "LD": 258,
+      "YR": 259,
+      "AAV": 260,
+      "TAL": 261,
+      "ATS": 262,
+      "KF": 263,
+      "GGG": 264,
+      "CP": 265,
+      "ADV": 266,
+      "SE": 267,
+      "LSG": 268,
+      "AEL": 269,
+      "AGL": 270,
+      "SF": 271,
+      "YN": 272,
+      "DH": 273,
+      "PN": 274,
+      "TAP": 275,
+      "VN": 276,
+      "ADL": 277,
+      "LLL": 278,
+      "SSG": 279,
+      "ASL": 280,
+      "SSL": 281,
+      "FR": 282,
+      "YE": 283,
+      "IR": 284,
+      "YK": 285,
+      "ARL": 286,
+      "DM": 287,
+      "HH": 288,
+      "WG": 289,
+      "FP": 290,
+      "VK": 291,
+      "EEL": 292,
+      "IQ": 293,
+      "AAS": 294,
+      "LSL": 295,
+      "CR": 296,
+      "TDG": 297,
+      "TSG": 298,
+      "MP": 299,
+      "ALG": 300,
+      "ANG": 301,
+      "AVL": 302,
+      "HR": 303,
+      "CI": 304,
+      "AAAA": 305,
+      "VF": 306,
+      "TTG": 307,
+      "ME": 308,
+      "YF": 309,
+      "SN": 310,
+      "MK": 311,
+      "TAE": 312,
+      "AEE": 313,
+      "YP": 314,
+      "HE": 315,
+      "MI": 316,
+      "CE": 317,
+      "VQ": 318,
+      "TAT": 319,
+      "YQ": 320,
+      "ATP": 321,
+      "TTS": 322,
+      "GE": 323,
+      "LLS": 324,
+      "AGS": 325,
+      "TEE": 326,
+      "FQ": 327,
+      "AGV": 328,
+      "GR": 329,
+      "WV": 330,
+      "XXXX": 331,
+      "TTL": 332,
+      "TVS": 333,
+      "GI": 334,
+      "CD": 335,
+      "TSL": 336,
+      "HD": 337,
+      "TDV": 338,
+      "MD": 339,
+      "ATI": 340,
+      "CK": 341,
+      "ATD": 342,
+      "TTV": 343,
+      "TGL": 344,
+      "MR": 345,
+      "TAD": 346,
+      "HF": 347,
+      "DGS": 348,
+      "SGS": 349,
+      "HQ": 350,
+      "CQ": 351,
+      "GGS": 352,
+      "WR": 353,
+      "IF": 354,
+      "LLG": 355,
+      "TDL": 356,
+      "DSDS": 357,
+      "AQL": 358,
+      "DVL": 359,
+      "MN": 360,
+      "MQ": 361,
+      "ASV": 362,
+      "TGS": 363,
+      "TVL": 364,
+      "HK": 365,
+      "GN": 366,
+      "DGL": 367,
+      "IY": 368,
+      "TEL": 369,
+      "DW": 370,
+      "TAI": 371,
+      "GP": 372,
+      "AVV": 373,
+      "GGL": 374,
+      "EVL": 375,
+      "SGL": 376,
+      "CN": 377,
+      "FY": 378,
+      "DAA": 379,
+      "SSV": 380,
+      "HY": 381,
+      "AVG": 382,
+      "HN": 383,
+      "PY": 384,
+      "SR": 385,
+      "SK": 386,
+      "APG": 387,
+      "ALV": 388,
+      "DGG": 389,
+      "SVL": 390,
+      "APL": 391,
+      "RVL": 392,
+      "LSV": 393,
+      "TEV": 394,
+      "RM": 395,
+      "ALP": 396,
+      "RW": 397,
+      "AVS": 398,
+      "DGV": 399,
+      "TPL": 400,
+      "AKL": 401,
+      "TSV": 402,
+      "AAP": 403,
+      "VVL": 404,
+      "ALR": 405,
+      "EC": 406,
+      "TEI": 407,
+      "TTTT": 408,
+      "SVS": 409,
+      "TAR": 410,
+      "TNG": 411,
+      "TEG": 412,
+      "EW": 413,
+      "AEG": 414,
+      "APV": 415,
+      "QM": 416,
+      "TVV": 417,
+      "TAQ": 418,
+      "PH": 419,
+      "CF": 420,
+      "ANL": 421,
+      "TES": 422,
+      "KM": 423,
+      "TPS": 424,
+      "GGV": 425,
+      "RLV": 426,
+      "NM": 427,
+      "PVL": 428,
+      "TLG": 429,
+      "DAG": 430,
+      "SSSS": 431,
+      "DC": 432,
+      "WI": 433,
+      "ELL": 434,
+      "EGL": 435,
+      "RLL": 436,
+      "ELG": 437,
+      "ANV": 438,
+      "SY": 439,
+      "EAA": 440,
+      "NH": 441,
+      "TVG": 442,
+      "SLV": 443,
+      "QQQQ": 444,
+      "PAA": 445,
+      "WQ": 446,
+      "EAL": 447,
+      "KH": 448,
+      "TNV": 449,
+      "SQ": 450,
+      "TGV": 451,
+      "MF": 452,
+      "DAL": 453,
+      "PAG": 454,
+      "DAV": 455,
+      "DLS": 456,
+      "TPV": 457,
+      "AEV": 458,
+      "DIL": 459,
+      "TNL": 460,
+      "PSP": 461,
+      "TLV": 462,
+      "WN": 463,
+      "WK": 464,
+      "TRL": 465,
+      "TTP": 466,
+      "ARG": 467,
+      "RC": 468,
+      "DLL": 469,
+      "DLV": 470,
+      "TAK": 471,
+      "PGL": 472,
+      "DLG": 473,
+      "SLG": 474,
+      "RAA": 475,
+      "DVV": 476,
+      "NGG": 477,
+      "SGV": 478,
+      "SSI": 479,
+      "TIS": 480,
+      "TPG": 481,
+      "RGL": 482,
+      "NC": 483,
+      "EEV": 484,
+      "TEK": 485,
+      "VVV": 486,
+      "FH": 487,
+      "YH": 488,
+      "EIL": 489,
+      "TAN": 490,
+      "NGL": 491,
+      "APS": 492,
+      "IH": 493,
+      "WE": 494,
+      "TQL": 495,
+      "RLG": 496,
+      "VVG": 497,
+      "TNI": 498,
+      "TLP": 499,
+      "SSP": 500,
+      "TTI": 501,
+      "DAS": 502,
+      "TKL": 503,
+      "NVL": 504,
+      "QLL": 505,
+      "TDS": 506,
+      "AIL": 507,
+      "AKK": 508,
+      "PLP": 509,
+      "QC": 510,
+      "DSL": 511,
+      "ELV": 512,
+      "DVS": 513,
+      "ANI": 514,
+      "NW": 515,
+      "ANS": 516,
+      "QW": 517,
+      "TSP": 518,
+      "QAA": 519,
+      "TNS": 520,
+      "DSS": 521,
+      "TAF": 522,
+      "PGS": 523,
+      "DSG": 524,
+      "VVS": 525,
+      "PM": 526,
+      "GK": 527,
+      "DAD": 528,
+      "LLV": 529,
+      "AGN": 530,
+      "LSP": 531,
+      "ESL": 532,
+      "NGS": 533,
+      "DDV": 534,
+      "EAV": 535,
+      "DEL": 536,
+      "TIL": 537,
+      "SVG": 538,
+      "NGV": 539,
+      "EEI": 540,
+      "AIG": 541,
+      "XXXXXXXX": 542,
+      "DDL": 543,
+      "AGI": 544,
+      "ASI": 545,
+      "CY": 546,
+      "FM": 547,
+      "AAR": 548,
+      "AAE": 549,
+      "AAI": 550,
+      "NIL": 551,
+      "TKK": 552,
+      "TSI": 553,
+      "TKV": 554,
+      "TVN": 555,
+      "DTL": 556,
+      "MY": 557,
+      "QGL": 558,
+      "AGE": 559,
+      "DEE": 560,
+      "ADS": 561,
+      "AGR": 562,
+      "SLP": 563,
+      "AES": 564,
+      "ASP": 565,
+      "LLE": 566,
+      "LSI": 567,
+      "DTV": 568,
+      "DTT": 569,
+      "PVG": 570,
+      "GGGG": 571,
+      "TRV": 572,
+      "PGV": 573,
+      "RAL": 574,
+      "PVS": 575,
+      "EVS": 576,
+      "AYL": 577,
+      "KC": 578,
+      "ADI": 579,
+      "WF": 580,
+      "NSS": 581,
+      "TQV": 582,
+      "DSI": 583,
+      "QAL": 584,
+      "PPP": 585,
+      "ARV": 586,
+      "EEE": 587,
+      "NTL": 588,
+      "DDG": 589,
+      "PDG": 590,
+      "RLR": 591,
+      "LLR": 592,
+      "DSV": 593,
+      "TTE": 594,
+      "EEG": 595,
+      "CH": 596,
+      "KW": 597,
+      "TKI": 598,
+      "AEI": 599,
+      "LSE": 600,
+      "MM": 601,
+      "AQG": 602,
+      "NVS": 603,
+      "DLP": 604,
+      "EGV": 605,
+      "DEV": 606,
+      "QQL": 607,
+      "EKL": 608,
+      "YW": 609,
+      "REL": 610,
+      "RIL": 611,
+      "LLP": 612,
+      "PLV": 613,
+      "DIS": 614,
+      "TAY": 615,
+      "AFL": 616,
+      "PVP": 617,
+      "QSL": 618,
+      "TVP": 619,
+      "NAA": 620,
+      "AIS": 621,
+      "ERL": 622,
+      "NLG": 623,
+      "RAV": 624,
+      "AQV": 625,
+      "EES": 626,
+      "DGI": 627,
+      "TIG": 628,
+      "EGS": 629,
+      "APP": 630,
+      "PLG": 631,
+      "LLK": 632,
+      "NLS": 633,
+      "TDI": 634,
+      "NLV": 635,
+      "QLV": 636,
+      "TPP": 637,
+      "RLS": 638,
+      "EEK": 639,
+      "DRL": 640,
+      "ETL": 641,
+      "VLG": 642,
+      "TRS": 643,
+      "TGN": 644,
+      "TQI": 645,
+      "TRG": 646,
+      "AHL": 647,
+      "ELP": 648,
+      "FC": 649,
+      "TNN": 650,
+      "DSDSDSDS": 651,
+      "TGI": 652,
+      "DTS": 653,
+      "RVS": 654,
+      "EIS": 655,
+      "MH": 656,
+      "KGL": 657,
+      "TIV": 658,
+      "TVE": 659,
+      "TLN": 660,
+      "DDI": 661,
+      "QLG": 662,
+      "DAE": 663,
+      "TVI": 664,
+      "DVG": 665,
+      "TQG": 666,
+      "FGL": 667,
+      "TKS": 668,
+      "QAV": 669,
+      "ARS": 670,
+      "KLV": 671,
+      "NIS": 672,
+      "DAP": 673,
+      "TQS": 674,
+      "DLI": 675,
+      "RGG": 676,
+      "WP": 677,
+      "AIV": 678,
+      "NSL": 679,
+      "DGK": 680,
+      "YM": 681,
+      "ESI": 682,
+      "RRL": 683,
+      "RSL": 684,
+      "ELS": 685,
+      "PSS": 686,
+      "EGG": 687,
+      "EKV": 688,
+      "QLS": 689,
+      "NTT": 690,
+      "RSS": 691,
+      "ESS": 692,
+      "ETT": 693,
+      "QSS": 694,
+      "TRI": 695,
+      "PGG": 696,
+      "EAE": 697,
+      "KIL": 698,
+      "FSS": 699,
+      "EIV": 700,
+      "SIL": 701,
+      "TDP": 702,
+      "NSG": 703,
+      "RGS": 704,
+      "TRR": 705,
+      "ADP": 706,
+      "QVV": 707,
+      "RVG": 708,
+      "TQQ": 709,
+      "DIV": 710,
+      "NVG": 711,
+      "KEL": 712,
+      "TLI": 713,
+      "QRL": 714,
+      "EAG": 715,
+      "AKV": 716,
+      "QIL": 717,
+      "AVI": 718,
+      "NLL": 719,
+      "NAG": 720,
+      "YC": 721,
+      "DTG": 722,
+      "NAS": 723,
+      "RAG": 724,
+      "NSI": 725,
+      "TVTV": 726,
+      "QGG": 727,
+      "SGI": 728,
+      "KGV": 729,
+      "HM": 730,
+      "QVL": 731,
+      "FGG": 732,
+      "EVV": 733,
+      "ESV": 734,
+      "QEL": 735,
+      "KEE": 736,
+      "DVI": 737,
+      "ETV": 738,
+      "PC": 739,
+      "AFG": 740,
+      "EKI": 741,
+      "ALI": 742,
+      "HC": 743,
+      "EQL": 744,
+      "TFS": 745,
+      "TPI": 746,
+      "SIS": 747,
+      "FW": 748,
+      "RGV": 749,
+      "NIG": 750,
+      "QTL": 751,
+      "EAS": 752,
+      "KLK": 753,
+      "NLI": 754,
+      "FSG": 755,
+      "RSG": 756,
+      "FDL": 757,
+      "PSI": 758,
+      "AVP": 759,
+      "TKG": 760,
+      "EDL": 761,
+      "KKKK": 762,
+      "RAR": 763,
+      "QAG": 764,
+      "EKS": 765,
+      "EVG": 766,
+      "TVK": 767,
+      "NAL": 768,
+      "DAI": 769,
+      "VGL": 770,
+      "NGI": 771,
+      "DEI": 772,
+      "AKG": 773,
+      "TLK": 774,
+      "EDV": 775,
+      "ETS": 776,
+      "ESG": 777,
+      "PTT": 778,
+      "WY": 779,
+      "KAL": 780,
+      "TVD": 781,
+      "KLG": 782,
+      "RDL": 783,
+      "QSV": 784,
+      "YLG": 785,
+      "LSF": 786,
+      "DQL": 787,
+      "TSGS": 788,
+      "CSS": 789,
+      "DDS": 790,
+      "DAR": 791,
+      "PAP": 792,
+      "RVV": 793,
+      "NTS": 794,
+      "PAV": 795,
+      "QAS": 796,
+      "AQS": 797,
+      "LSK": 798,
+      "EDG": 799,
+      "CC": 800,
+      "SLI": 801,
+      "DTI": 802,
+      "NTG": 803,
+      "RRS": 804,
+      "ELI": 805,
+      "NAV": 806,
+      "EVI": 807,
+      "QSG": 808,
+      "KLL": 809,
+      "DNL": 810,
+      "GGI": 811,
+      "KSS": 812,
+      "AAF": 813,
+      "RLP": 814,
+      "QTV": 815,
+      "QVS": 816,
+      "PVI": 817,
+      "VLV": 818,
+      "AHG": 819,
+      "TFL": 820,
+      "NSV": 821,
+      "DPS": 822,
+      "AKS": 823,
+      "RAE": 824,
+      "PSG": 825,
+      "QTT": 826,
+      "EKG": 827,
+      "KLI": 828,
+      "ENL": 829,
+      "RTL": 830,
+      "KKK": 831,
+      "LLI": 832,
+      "DRV": 833,
+      "FTG": 834,
+      "DAT": 835,
+      "NVV": 836,
+      "DVP": 837,
+      "AFV": 838,
+      "KSL": 839,
+      "DIG": 840,
+      "HW": 841,
+      "EPG": 842,
+      "DTD": 843,
+      "RRG": 844,
+      "PIL": 845,
+      "AAK": 846,
+      "FLG": 847,
+      "EII": 848,
+      "QGS": 849,
+      "NNI": 850,
+      "NNL": 851,
+      "NVI": 852,
+      "SIG": 853,
+      "TGK": 854,
+      "RTV": 855,
+      "NDG": 856,
+      "KKS": 857,
+      "AAQ": 858,
+      "FTV": 859,
+      "NDL": 860,
+      "DES": 861,
+      "RSV": 862,
+      "LLQ": 863,
+      "RDG": 864,
+      "ALK": 865,
+      "DEG": 866,
+      "ALE": 867,
+      "PEP": 868,
+      "TGP": 869,
+      "RAS": 870,
+      "ELK": 871,
+      "GLG": 872,
+      "DPV": 873,
+      "EKP": 874,
+      "DKL": 875,
+      "REE": 876,
+      "RVI": 877,
+      "NTV": 878,
+      "KVS": 879,
+      "DII": 880,
+      "KKI": 881,
+      "AML": 882,
+      "TYL": 883,
+      "EIG": 884,
+      "FSL": 885,
+      "AYV": 886,
+      "FAA": 887,
+      "QVG": 888,
+      "SVI": 889,
+      "ETI": 890,
+      "YGG": 891,
+      "VGV": 892,
+      "TYS": 893,
+      "KKV": 894,
+      "QTI": 895,
+      "NTI": 896,
+      "FGS": 897,
+      "AFS": 898,
+      "EEEE": 899,
+      "RLI": 900,
+      "RDV": 901,
+      "VGS": 902,
+      "TFG": 903,
+      "AGP": 904,
+      "RQL": 905,
+      "TRE": 906,
+      "RPG": 907,
+      "KIS": 908,
+      "ALQ": 909,
+      "HLL": 910,
+      "ARI": 911,
+      "EAP": 912,
+      "NNS": 913,
+      "QTS": 914,
+      "EGI": 915,
+      "FDV": 916,
+      "EPV": 917,
+      "TRP": 918,
+      "VVI": 919,
+      "NIV": 920,
+      "KAG": 921,
+      "NDV": 922,
+      "DNG": 923,
+      "QQV": 924,
+      "KKG": 925,
+      "QSI": 926,
+      "NEL": 927,
+      "QGV": 928,
+      "DNV": 929,
+      "KLS": 930,
+      "TAH": 931,
+      "QEE": 932,
+      "SVP": 933,
+      "ASF": 934,
+      "API": 935,
+      "KDL": 936,
+      "QAE": 937,
+      "EDI": 938,
+      "RPV": 939,
+      "RPL": 940,
+      "WH": 941,
+      "YTL": 942,
+      "DPG": 943,
+      "NNG": 944,
+      "YSG": 945,
+      "DPL": 946,
+      "RIV": 947,
+      "NII": 948,
+      "EAI": 949,
+      "ETG": 950,
+      "RIS": 951,
+      "EPL": 952,
+      "DKV": 953,
+      "PLL": 954,
+      "YTV": 955,
+      "PIG": 956,
+      "FTL": 957,
+      "PSL": 958,
+      "AQI": 959,
+      "WM": 960,
+      "NAT": 961,
+      "THL": 962,
+      "DNS": 963,
+      "QDL": 964,
+      "QAR": 965,
+      "QPG": 966,
+      "PAL": 967,
+      "CM": 968,
+      "YTG": 969,
+      "TFV": 970,
+      "YSS": 971,
+      "RTG": 972,
+      "QIS": 973,
+      "FVS": 974,
+      "PAS": 975,
+      "KEK": 976,
+      "QPV": 977,
+      "KDG": 978,
+      "KAV": 979,
+      "TII": 980,
+      "QLI": 981,
+      "FAG": 982,
+      "DAN": 983,
+      "AKI": 984,
+      "RIG": 985,
+      "NDI": 986,
+      "QPL": 987,
+      "RTT": 988,
+      "PLS": 989,
+      "ERV": 990,
+      "PW": 991,
+      "NAI": 992,
+      "QTG": 993,
+      "QKL": 994,
+      "TYG": 995,
+      "EKK": 996,
+      "EAR": 997,
+      "TAM": 998,
+      "KVL": 999,
+      "FDG": 1000,
+      "KTL": 1001,
+      "KGS": 1002,
+      "XXXXXXXXXXXXXXXX": 1003,
+      "PTE": 1004,
+      "REV": 1005,
+      "DKI": 1006,
+      "QQG": 1007,
+      "DAF": 1008,
+      "KKL": 1009,
+      "DAQ": 1010,
+      "KAA": 1011,
+      "PSV": 1012,
+      "FSV": 1013,
+      "DNI": 1014,
+      "FGV": 1015,
+      "DTP": 1016,
+      "PIS": 1017,
+      "FLS": 1018,
+      "PEE": 1019,
+      "QEV": 1020,
+      "KEI": 1021,
+      "GLV": 1022,
+      "FAE": 1023
+    },
+    "merges": [
+      "A A",
+      "S S",
+      "T T",
+      "G G",
+      "L L",
+      "A G",
+      "L S",
+      "T V",
+      "A V",
+      "A L",
+      "A S",
+      "D G",
+      "T G",
+      "E E",
+      "T L",
+      "T S",
+      "D V",
+      "A E",
+      "D L",
+      "S G",
+      "A P",
+      "R L",
+      "V V",
+      "S L",
+      "T P",
+      "T I",
+      "E L",
+      "S V",
+      "A D",
+      "N G",
+      "A R",
+      "E V",
+      "Q L",
+      "T D",
+      "A I",
+      "N L",
+      "E G",
+      "P V",
+      "K L",
+      "P G",
+      "E I",
+      "N V",
+      "D I",
+      "P L",
+      "D S",
+      "K K",
+      "R V",
+      "N I",
+      "A Q",
+      "P S",
+      "A T",
+      "E K",
+      "E S",
+      "R G",
+      "Q Q",
+      "N S",
+      "D D",
+      "A K",
+      "R R",
+      "V L",
+      "A F",
+      "T F",
+      "R I",
+      "P P",
+      "Q V",
+      "T E",
+      "A N",
+      "S I",
+      "K V",
+      "Q G",
+      "K I",
+      "R S",
+      "T Y",
+      "D P",
+      "N N",
+      "K G",
+      "D E",
+      "Q I",
+      "F G",
+      "V G",
+      "Y L",
+      "F L",
+      "T N",
+      "P I",
+      "K S",
+      "Q S",
+      "P E",
+      "A Y",
+      "H L",
+      "R E",
+      "F S",
+      "F V",
+      "T K",
+      "G L",
+      "V S",
+      "T Q",
+      "D N",
+      "I L",
+      "R P",
+      "G S",
+      "K E",
+      "I S",
+      "D F",
+      "T R",
+      "D R",
+      "D Y",
+      "A H",
+      "D K",
+      "I G",
+      "Q E",
+      "A M",
+      "Y G",
+      "N E",
+      "I V",
+      "Y S",
+      "N P",
+      "E R",
+      "Q P",
+      "Y V",
+      "M L",
+      "T A",
+      "Q R",
+      "G V",
+      "N D",
+      "K P",
+      "F E",
+      "F I",
+      "N K",
+      "H S",
+      "H G",
+      "Q K",
+      "C L",
+      "H V",
+      "N Y",
+      "I E",
+      "D Q",
+      "W L",
+      "R K",
+      "C S",
+      "N F",
+      "R D",
+      "E P",
+      "R F",
+      "AA L",
+      "E D",
+      "I I",
+      "T M",
+      "T C",
+      "N Q",
+      "T H",
+      "A GG",
+      "F D",
+      "AA G",
+      "R Q",
+      "A C",
+      "P D",
+      "V I",
+      "E Q",
+      "L G",
+      "Y I",
+      "A W",
+      "M S",
+      "M V",
+      "K D",
+      "L V",
+      "SS S",
+      "N R",
+      "C G",
+      "H I",
+      "P K",
+      "T W",
+      "R Y",
+      "E F",
+      "E N",
+      "A DG",
+      "A LL",
+      "P Q",
+      "E Y",
+      "C V",
+      "T AA",
+      "P F",
+      "X X",
+      "T SS",
+      "M G",
+      "K Q",
+      "I D",
+      "P R",
+      "T LS",
+      "A SS",
+      "Q D",
+      "R N",
+      "W S",
+      "R H",
+      "F K",
+      "V D",
+      "V E",
+      "K N",
+      "T GG",
+      "F F",
+      "AS G",
+      "Q N",
+      "A TT",
+      "Q Y",
+      "H P",
+      "A TG",
+      "A TV",
+      "K Y",
+      "V P",
+      "A LS",
+      "Q F",
+      "I N",
+      "T AG",
+      "I K",
+      "T AS",
+      "S P",
+      "Y Y",
+      "F N",
+      "L P",
+      "I P",
+      "Y D",
+      "E H",
+      "T AV",
+      "K R",
+      "S D",
+      "V R",
+      "A TL",
+      "G D",
+      "E M",
+      "T LL",
+      "Q H",
+      "L D",
+      "Y R",
+      "AA V",
+      "T AL",
+      "A TS",
+      "K F",
+      "GG G",
+      "C P",
+      "A DV",
+      "S E",
+      "LS G",
+      "AE L",
+      "AG L",
+      "S F",
+      "Y N",
+      "D H",
+      "P N",
+      "T AP",
+      "V N",
+      "A DL",
+      "LL L",
+      "SS G",
+      "AS L",
+      "SS L",
+      "F R",
+      "Y E",
+      "I R",
+      "Y K",
+      "A RL",
+      "D M",
+      "H H",
+      "W G",
+      "F P",
+      "V K",
+      "EE L",
+      "I Q",
+      "AA S",
+      "LS L",
+      "C R",
+      "T DG",
+      "TS G",
+      "M P",
+      "AL G",
+      "A NG",
+      "AV L",
+      "H R",
+      "C I",
+      "AA AA",
+      "V F",
+      "TT G",
+      "M E",
+      "Y F",
+      "S N",
+      "M K",
+      "T AE",
+      "A EE",
+      "Y P",
+      "H E",
+      "M I",
+      "C E",
+      "V Q",
+      "T AT",
+      "Y Q",
+      "A TP",
+      "TT S",
+      "G E",
+      "LL S",
+      "AG S",
+      "T EE",
+      "F Q",
+      "AG V",
+      "G R",
+      "W V",
+      "XX XX",
+      "TT L",
+      "TV S",
+      "G I",
+      "C D",
+      "TS L",
+      "H D",
+      "T DV",
+      "M D",
+      "A TI",
+      "C K",
+      "A TD",
+      "TT V",
+      "TG L",
+      "M R",
+      "T AD",
+      "H F",
+      "DG S",
+      "SG S",
+      "H Q",
+      "C Q",
+      "GG S",
+      "W R",
+      "I F",
+      "LL G",
+      "T DL",
+      "DS DS",
+      "A QL",
+      "DV L",
+      "M N",
+      "M Q",
+      "AS V",
+      "TG S",
+      "TV L",
+      "H K",
+      "G N",
+      "DG L",
+      "I Y",
+      "T EL",
+      "D W",
+      "T AI",
+      "G P",
+      "AV V",
+      "GG L",
+      "EV L",
+      "SG L",
+      "C N",
+      "F Y",
+      "D AA",
+      "SS V",
+      "H Y",
+      "AV G",
+      "H N",
+      "P Y",
+      "S R",
+      "S K",
+      "AP G",
+      "AL V",
+      "D GG",
+      "SV L",
+      "AP L",
+      "RV L",
+      "LS V",
+      "T EV",
+      "R M",
+      "AL P",
+      "R W",
+      "AV S",
+      "DG V",
+      "TP L",
+      "A KL",
+      "TS V",
+      "AA P",
+      "VV L",
+      "AL R",
+      "E C",
+      "T EI",
+      "TT TT",
+      "SV S",
+      "T AR",
+      "T NG",
+      "T EG",
+      "E W",
+      "AE G",
+      "AP V",
+      "Q M",
+      "TV V",
+      "T AQ",
+      "P H",
+      "C F",
+      "A NL",
+      "T ES",
+      "K M",
+      "TP S",
+      "GG V",
+      "RL V",
+      "N M",
+      "PV L",
+      "TL G",
+      "D AG",
+      "SS SS",
+      "D C",
+      "W I",
+      "E LL",
+      "EG L",
+      "R LL",
+      "EL G",
+      "A NV",
+      "S Y",
+      "E AA",
+      "N H",
+      "TV G",
+      "SL V",
+      "QQ QQ",
+      "P AA",
+      "W Q",
+      "E AL",
+      "K H",
+      "T NV",
+      "S Q",
+      "TG V",
+      "M F",
+      "D AL",
+      "P AG",
+      "D AV",
+      "D LS",
+      "TP V",
+      "AE V",
+      "DI L",
+      "T NL",
+      "PS P",
+      "TL V",
+      "W N",
+      "W K",
+      "T RL",
+      "TT P",
+      "AR G",
+      "R C",
+      "D LL",
+      "DL V",
+      "T AK",
+      "PG L",
+      "DL G",
+      "SL G",
+      "R AA",
+      "DV V",
+      "N GG",
+      "SG V",
+      "SS I",
+      "TI S",
+      "TP G",
+      "RG L",
+      "N C",
+      "EE V",
+      "T EK",
+      "VV V",
+      "F H",
+      "Y H",
+      "EI L",
+      "T AN",
+      "NG L",
+      "AP S",
+      "I H",
+      "W E",
+      "T QL",
+      "RL G",
+      "VV G",
+      "T NI",
+      "TL P",
+      "SS P",
+      "TT I",
+      "D AS",
+      "T KL",
+      "NV L",
+      "Q LL",
+      "TD S",
+      "AI L",
+      "A KK",
+      "PL P",
+      "Q C",
+      "D SL",
+      "EL V",
+      "DV S",
+      "A NI",
+      "N W",
+      "A NS",
+      "Q W",
+      "TS P",
+      "Q AA",
+      "T NS",
+      "D SS",
+      "T AF",
+      "PG S",
+      "D SG",
+      "VV S",
+      "P M",
+      "G K",
+      "D AD",
+      "LL V",
+      "AG N",
+      "LS P",
+      "E SL",
+      "NG S",
+      "D DV",
+      "E AV",
+      "D EL",
+      "TI L",
+      "SV G",
+      "NG V",
+      "EE I",
+      "AI G",
+      "XXXX XXXX",
+      "D DL",
+      "AG I",
+      "AS I",
+      "C Y",
+      "F M",
+      "AA R",
+      "AA E",
+      "AA I",
+      "NI L",
+      "T KK",
+      "TS I",
+      "T KV",
+      "TV N",
+      "D TL",
+      "M Y",
+      "QG L",
+      "AG E",
+      "D EE",
+      "AD S",
+      "AG R",
+      "SL P",
+      "AE S",
+      "AS P",
+      "LL E",
+      "LS I",
+      "D TV",
+      "D TT",
+      "PV G",
+      "GG GG",
+      "T RV",
+      "PG V",
+      "R AL",
+      "PV S",
+      "EV S",
+      "A YL",
+      "K C",
+      "AD I",
+      "W F",
+      "N SS",
+      "T QV",
+      "DS I",
+      "Q AL",
+      "PP P",
+      "AR V",
+      "EE E",
+      "N TL",
+      "D DG",
+      "P DG",
+      "RL R",
+      "LL R",
+      "D SV",
+      "TT E",
+      "EE G",
+      "C H",
+      "K W",
+      "T KI",
+      "AE I",
+      "LS E",
+      "M M",
+      "AQ G",
+      "NV S",
+      "DL P",
+      "EG V",
+      "D EV",
+      "Q QL",
+      "E KL",
+      "Y W",
+      "R EL",
+      "RI L",
+      "LL P",
+      "PL V",
+      "DI S",
+      "T AY",
+      "AF L",
+      "PV P",
+      "Q SL",
+      "TV P",
+      "N AA",
+      "AI S",
+      "E RL",
+      "NL G",
+      "R AV",
+      "AQ V",
+      "EE S",
+      "DG I",
+      "TI G",
+      "EG S",
+      "AP P",
+      "PL G",
+      "LL K",
+      "N LS",
+      "TD I",
+      "NL V",
+      "QL V",
+      "TP P",
+      "R LS",
+      "EE K",
+      "D RL",
+      "E TL",
+      "VL G",
+      "T RS",
+      "TG N",
+      "T QI",
+      "T RG",
+      "A HL",
+      "EL P",
+      "F C",
+      "T NN",
+      "DSDS DSDS",
+      "TG I",
+      "D TS",
+      "RV S",
+      "EI S",
+      "M H",
+      "KG L",
+      "TI V",
+      "TV E",
+      "TL N",
+      "D DI",
+      "QL G",
+      "D AE",
+      "TV I",
+      "DV G",
+      "T QG",
+      "FG L",
+      "T KS",
+      "Q AV",
+      "AR S",
+      "KL V",
+      "NI S",
+      "D AP",
+      "T QS",
+      "DL I",
+      "R GG",
+      "W P",
+      "AI V",
+      "N SL",
+      "DG K",
+      "Y M",
+      "ES I",
+      "R RL",
+      "R SL",
+      "E LS",
+      "P SS",
+      "E GG",
+      "EK V",
+      "Q LS",
+      "N TT",
+      "R SS",
+      "E SS",
+      "E TT",
+      "Q SS",
+      "T RI",
+      "P GG",
+      "E AE",
+      "KI L",
+      "F SS",
+      "EI V",
+      "SI L",
+      "TD P",
+      "N SG",
+      "RG S",
+      "T RR",
+      "AD P",
+      "Q VV",
+      "RV G",
+      "T QQ",
+      "DI V",
+      "NV G",
+      "K EL",
+      "TL I",
+      "Q RL",
+      "E AG",
+      "AK V",
+      "QI L",
+      "AV I",
+      "N LL",
+      "N AG",
+      "Y C",
+      "D TG",
+      "N AS",
+      "R AG",
+      "NS I",
+      "TV TV",
+      "Q GG",
+      "SG I",
+      "KG V",
+      "H M",
+      "Q VL",
+      "F GG",
+      "E VV",
+      "E SV",
+      "Q EL",
+      "K EE",
+      "DV I",
+      "E TV",
+      "P C",
+      "AF G",
+      "EK I",
+      "AL I",
+      "H C",
+      "E QL",
+      "TF S",
+      "TP I",
+      "SI S",
+      "F W",
+      "RG V",
+      "NI G",
+      "Q TL",
+      "E AS",
+      "KL K",
+      "NL I",
+      "F SG",
+      "R SG",
+      "F DL",
+      "PS I",
+      "AV P",
+      "T KG",
+      "E DL",
+      "KK KK",
+      "R AR",
+      "Q AG",
+      "EK S",
+      "EV G",
+      "TV K",
+      "N AL",
+      "D AI",
+      "VG L",
+      "NG I",
+      "D EI",
+      "AK G",
+      "TL K",
+      "E DV",
+      "E TS",
+      "E SG",
+      "P TT",
+      "W Y",
+      "K AL",
+      "TV D",
+      "KL G",
+      "R DL",
+      "Q SV",
+      "YL G",
+      "LS F",
+      "D QL",
+      "TS GS",
+      "C SS",
+      "D DS",
+      "D AR",
+      "P AP",
+      "R VV",
+      "N TS",
+      "P AV",
+      "Q AS",
+      "AQ S",
+      "LS K",
+      "E DG",
+      "C C",
+      "SL I",
+      "D TI",
+      "N TG",
+      "RR S",
+      "EL I",
+      "N AV",
+      "EV I",
+      "Q SG",
+      "K LL",
+      "D NL",
+      "GG I",
+      "K SS",
+      "AA F",
+      "RL P",
+      "Q TV",
+      "QV S",
+      "PV I",
+      "VL V",
+      "AH G",
+      "TF L",
+      "N SV",
+      "D PS",
+      "AK S",
+      "R AE",
+      "P SG",
+      "Q TT",
+      "EK G",
+      "KL I",
+      "E NL",
+      "R TL",
+      "KK K",
+      "LL I",
+      "D RV",
+      "F TG",
+      "D AT",
+      "N VV",
+      "DV P",
+      "AF V",
+      "K SL",
+      "DI G",
+      "H W",
+      "E PG",
+      "D TD",
+      "R RG",
+      "PI L",
+      "AA K",
+      "FL G",
+      "EI I",
+      "QG S",
+      "N NI",
+      "N NL",
+      "NV I",
+      "SI G",
+      "TG K",
+      "R TV",
+      "N DG",
+      "KK S",
+      "AA Q",
+      "F TV",
+      "N DL",
+      "D ES",
+      "R SV",
+      "LL Q",
+      "R DG",
+      "AL K",
+      "D EG",
+      "AL E",
+      "PE P",
+      "TG P",
+      "R AS",
+      "EL K",
+      "GL G",
+      "D PV",
+      "EK P",
+      "D KL",
+      "R EE",
+      "RV I",
+      "N TV",
+      "KV S",
+      "DI I",
+      "KK I",
+      "AM L",
+      "TY L",
+      "EI G",
+      "F SL",
+      "AY V",
+      "F AA",
+      "QV G",
+      "SV I",
+      "E TI",
+      "Y GG",
+      "VG V",
+      "TY S",
+      "KK V",
+      "Q TI",
+      "N TI",
+      "FG S",
+      "AF S",
+      "EE EE",
+      "RL I",
+      "R DV",
+      "VG S",
+      "TF G",
+      "AG P",
+      "R QL",
+      "T RE",
+      "R PG",
+      "KI S",
+      "AL Q",
+      "H LL",
+      "AR I",
+      "E AP",
+      "N NS",
+      "Q TS",
+      "EG I",
+      "F DV",
+      "E PV",
+      "T RP",
+      "VV I",
+      "NI V",
+      "K AG",
+      "N DV",
+      "D NG",
+      "QQ V",
+      "KK G",
+      "Q SI",
+      "N EL",
+      "QG V",
+      "D NV",
+      "K LS",
+      "T AH",
+      "Q EE",
+      "SV P",
+      "AS F",
+      "AP I",
+      "K DL",
+      "Q AE",
+      "E DI",
+      "R PV",
+      "R PL",
+      "W H",
+      "Y TL",
+      "D PG",
+      "N NG",
+      "Y SG",
+      "D PL",
+      "RI V",
+      "NI I",
+      "E AI",
+      "E TG",
+      "RI S",
+      "E PL",
+      "D KV",
+      "P LL",
+      "Y TV",
+      "PI G",
+      "F TL",
+      "P SL",
+      "AQ I",
+      "W M",
+      "N AT",
+      "T HL",
+      "D NS",
+      "Q DL",
+      "Q AR",
+      "Q PG",
+      "P AL",
+      "C M",
+      "Y TG",
+      "TF V",
+      "Y SS",
+      "R TG",
+      "QI S",
+      "FV S",
+      "P AS",
+      "K EK",
+      "Q PV",
+      "K DG",
+      "K AV",
+      "TI I",
+      "QL I",
+      "F AG",
+      "D AN",
+      "AK I",
+      "RI G",
+      "N DI",
+      "Q PL",
+      "R TT",
+      "P LS",
+      "E RV",
+      "P W",
+      "N AI",
+      "Q TG",
+      "Q KL",
+      "TY G",
+      "E KK",
+      "E AR",
+      "T AM",
+      "K VL",
+      "F DG",
+      "K TL",
+      "KG S",
+      "XXXXXXXX XXXXXXXX",
+      "P TE",
+      "R EV",
+      "D KI",
+      "QQ G",
+      "D AF",
+      "K KL",
+      "D AQ",
+      "K AA",
+      "P SV",
+      "F SV",
+      "D NI",
+      "FG V",
+      "D TP",
+      "PI S",
+      "F LS",
+      "P EE",
+      "Q EV",
+      "K EI",
+      "GL V",
+      "F AE"
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,52 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}

trainer_state.json

@@ -0,0 +1,393 @@
+{
+  "best_metric": 5.84649133682251,
+  "best_model_checkpoint": "./results/models/checkpoint-23424",
+  "epoch": 12.0,
+  "eval_steps": 500,
+  "global_step": 23424,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.009948770491803278,
+      "loss": 5.9831,
+      "step": 500
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.009897540983606557,
+      "loss": 5.9498,
+      "step": 1000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.009846311475409836,
+      "loss": 5.9616,
+      "step": 1500
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 5.937765121459961,
+      "eval_runtime": 3.4294,
+      "eval_samples_per_second": 291.596,
+      "eval_steps_per_second": 0.583,
+      "step": 1952
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 0.009795081967213116,
+      "loss": 5.9374,
+      "step": 2000
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 0.009743852459016393,
+      "loss": 5.9391,
+      "step": 2500
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 0.009692622950819673,
+      "loss": 5.9287,
+      "step": 3000
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 0.00964139344262295,
+      "loss": 5.9149,
+      "step": 3500
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 5.907990455627441,
+      "eval_runtime": 3.1854,
+      "eval_samples_per_second": 313.928,
+      "eval_steps_per_second": 0.628,
+      "step": 3904
+    },
+    {
+      "epoch": 2.05,
+      "learning_rate": 0.00959016393442623,
+      "loss": 5.9179,
+      "step": 4000
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 0.009538934426229509,
+      "loss": 5.9105,
+      "step": 4500
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 0.009487704918032787,
+      "loss": 5.9086,
+      "step": 5000
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 0.009436475409836066,
+      "loss": 5.899,
+      "step": 5500
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 5.894569396972656,
+      "eval_runtime": 3.0831,
+      "eval_samples_per_second": 324.347,
+      "eval_steps_per_second": 0.649,
+      "step": 5856
+    },
+    {
+      "epoch": 3.07,
+      "learning_rate": 0.009385245901639345,
+      "loss": 5.8975,
+      "step": 6000
+    },
+    {
+      "epoch": 3.33,
+      "learning_rate": 0.009334016393442625,
+      "loss": 5.8936,
+      "step": 6500
+    },
+    {
+      "epoch": 3.59,
+      "learning_rate": 0.009282786885245902,
+      "loss": 5.8886,
+      "step": 7000
+    },
+    {
+      "epoch": 3.84,
+      "learning_rate": 0.00923155737704918,
+      "loss": 5.8882,
+      "step": 7500
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 5.879088878631592,
+      "eval_runtime": 3.2638,
+      "eval_samples_per_second": 306.393,
+      "eval_steps_per_second": 0.613,
+      "step": 7808
+    },
+    {
+      "epoch": 4.1,
+      "learning_rate": 0.009180327868852459,
+      "loss": 5.883,
+      "step": 8000
+    },
+    {
+      "epoch": 4.35,
+      "learning_rate": 0.009129098360655738,
+      "loss": 5.879,
+      "step": 8500
+    },
+    {
+      "epoch": 4.61,
+      "learning_rate": 0.009077868852459018,
+      "loss": 5.8752,
+      "step": 9000
+    },
+    {
+      "epoch": 4.87,
+      "learning_rate": 0.009026639344262295,
+      "loss": 5.8742,
+      "step": 9500
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 5.866305828094482,
+      "eval_runtime": 3.2,
+      "eval_samples_per_second": 312.496,
+      "eval_steps_per_second": 0.625,
+      "step": 9760
+    },
+    {
+      "epoch": 5.12,
+      "learning_rate": 0.008975409836065575,
+      "loss": 5.8721,
+      "step": 10000
+    },
+    {
+      "epoch": 5.38,
+      "learning_rate": 0.008924180327868852,
+      "loss": 5.8674,
+      "step": 10500
+    },
+    {
+      "epoch": 5.64,
+      "learning_rate": 0.008872950819672131,
+      "loss": 5.8722,
+      "step": 11000
+    },
+    {
+      "epoch": 5.89,
+      "learning_rate": 0.00882172131147541,
+      "loss": 5.8702,
+      "step": 11500
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 5.863623142242432,
+      "eval_runtime": 3.1064,
+      "eval_samples_per_second": 321.914,
+      "eval_steps_per_second": 0.644,
+      "step": 11712
+    },
+    {
+      "epoch": 6.15,
+      "learning_rate": 0.008770491803278688,
+      "loss": 5.8684,
+      "step": 12000
+    },
+    {
+      "epoch": 6.4,
+      "learning_rate": 0.008719262295081968,
+      "loss": 5.8664,
+      "step": 12500
+    },
+    {
+      "epoch": 6.66,
+      "learning_rate": 0.008668032786885245,
+      "loss": 5.8657,
+      "step": 13000
+    },
+    {
+      "epoch": 6.92,
+      "learning_rate": 0.008616803278688525,
+      "loss": 5.8616,
+      "step": 13500
+    },
+    {
+      "epoch": 7.0,
+      "eval_loss": 5.859433174133301,
+      "eval_runtime": 3.1303,
+      "eval_samples_per_second": 319.454,
+      "eval_steps_per_second": 0.639,
+      "step": 13664
+    },
+    {
+      "epoch": 7.17,
+      "learning_rate": 0.008565573770491804,
+      "loss": 5.8638,
+      "step": 14000
+    },
+    {
+      "epoch": 7.43,
+      "learning_rate": 0.008514344262295082,
+      "loss": 5.8628,
+      "step": 14500
+    },
+    {
+      "epoch": 7.68,
+      "learning_rate": 0.008463114754098361,
+      "loss": 5.8659,
+      "step": 15000
+    },
+    {
+      "epoch": 7.94,
+      "learning_rate": 0.008411885245901638,
+      "loss": 5.8659,
+      "step": 15500
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 5.862513065338135,
+      "eval_runtime": 3.0899,
+      "eval_samples_per_second": 323.637,
+      "eval_steps_per_second": 0.647,
+      "step": 15616
+    },
+    {
+      "epoch": 8.2,
+      "learning_rate": 0.008360655737704918,
+      "loss": 5.8653,
+      "step": 16000
+    },
+    {
+      "epoch": 8.45,
+      "learning_rate": 0.008309426229508197,
+      "loss": 5.8625,
+      "step": 16500
+    },
+    {
+      "epoch": 8.71,
+      "learning_rate": 0.008258196721311475,
+      "loss": 5.8673,
+      "step": 17000
+    },
+    {
+      "epoch": 8.97,
+      "learning_rate": 0.008206967213114754,
+      "loss": 5.8662,
+      "step": 17500
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 5.861349105834961,
+      "eval_runtime": 3.0909,
+      "eval_samples_per_second": 323.532,
+      "eval_steps_per_second": 0.647,
+      "step": 17568
+    },
+    {
+      "epoch": 9.22,
+      "learning_rate": 0.008155737704918033,
+      "loss": 5.8644,
+      "step": 18000
+    },
+    {
+      "epoch": 9.48,
+      "learning_rate": 0.008104508196721313,
+      "loss": 5.8615,
+      "step": 18500
+    },
+    {
+      "epoch": 9.73,
+      "learning_rate": 0.00805327868852459,
+      "loss": 5.8613,
+      "step": 19000
+    },
+    {
+      "epoch": 9.99,
+      "learning_rate": 0.00800204918032787,
+      "loss": 5.8591,
+      "step": 19500
+    },
+    {
+      "epoch": 10.0,
+      "eval_loss": 5.853126525878906,
+      "eval_runtime": 3.1333,
+      "eval_samples_per_second": 319.15,
+      "eval_steps_per_second": 0.638,
+      "step": 19520
+    },
+    {
+      "epoch": 10.25,
+      "learning_rate": 0.007950819672131147,
+      "loss": 5.8565,
+      "step": 20000
+    },
+    {
+      "epoch": 10.5,
+      "learning_rate": 0.007899590163934427,
+      "loss": 5.8547,
+      "step": 20500
+    },
+    {
+      "epoch": 10.76,
+      "learning_rate": 0.007848360655737706,
+      "loss": 5.8554,
+      "step": 21000
+    },
+    {
+      "epoch": 11.0,
+      "eval_loss": 5.850937843322754,
+      "eval_runtime": 3.092,
+      "eval_samples_per_second": 323.419,
+      "eval_steps_per_second": 0.647,
+      "step": 21472
+    },
+    {
+      "epoch": 11.01,
+      "learning_rate": 0.0077971311475409835,
+      "loss": 5.8557,
+      "step": 21500
+    },
+    {
+      "epoch": 11.27,
+      "learning_rate": 0.007745901639344263,
+      "loss": 5.8544,
+      "step": 22000
+    },
+    {
+      "epoch": 11.53,
+      "learning_rate": 0.007694672131147541,
+      "loss": 5.8536,
+      "step": 22500
+    },
+    {
+      "epoch": 11.78,
+      "learning_rate": 0.0076434426229508206,
+      "loss": 5.8531,
+      "step": 23000
+    },
+    {
+      "epoch": 12.0,
+      "eval_loss": 5.84649133682251,
+      "eval_runtime": 3.1942,
+      "eval_samples_per_second": 313.065,
+      "eval_steps_per_second": 0.626,
+      "step": 23424
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 97600,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 50,
+  "save_steps": 500,
+  "total_flos": 5.51042373648384e+17,
+  "train_batch_size": 512,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84f101afae5679ed1d12288811ec492221e83faad0b24f7186c4d410ed510c20
+size 4664