Upload folder using huggingface_hub

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "/data/niklas/sgpt2/sgpt2_m8x7_bs256_1253_token03",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

dataset_num_samples.json

@@ -0,0 +1 @@
+{"e5_train_ds_gpt4-retrieval_format.jsonl": 166602, "e5_train_ds_trivia_format.jsonl": 60296, "e5_train_ds_nli_format.jsonl": 275601, "e5_train_ds_gpt4-p2p_format.jsonl": 16842, "e5_train_ds_miracl_format.jsonl": 40203, "e5_train_ds_dureader_format.jsonl": 86395, "e5_train_ds_gpt4-p2s_format.jsonl": 121878, "e5_train_ds_fever_format.jsonl": 71257, "e5_train_ds_eli5_format.jsonl": 50293, "e5_train_ds_gpt4-s2s_format.jsonl": 13481, "e5_train_ds_msmarco-doc_format.jsonl": 71564, "e5_train_ds_t2ranking_format.jsonl": 112335, "e5_train_ds_quora_format.jsonl": 14926, "e5_train_ds_mrtydi_format.jsonl": 48729, "e5_train_ds_nq_format.jsonl": 71408, "e5_train_ds_squad_format.jsonl": 87599, "e5_train_ds_msmarco_format.jsonl": 244582, "e5_train_ds_s2orc_format.jsonl": 80000, "tuluv2.jsonl": 320860, "e5_train_ds_gpt4-bitext_format.jsonl": 89324, "e5_train_ds_gpt4-sts_format.jsonl": 98626, "e5_train_ds_hotpotqa_format.jsonl": 68659}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.37.2"
+}

model-00001-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0353c1aa5c778f2758231c72467e388073059388ee0a5b3b355c1071c69aa88a
+size 4892809584

model-00002-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:428831c8cd2cd0ffaa1287ffdfef63cb2614e873b48c28aad154a7f9d4b6adaa
+size 4983004016

model-00003-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f4b71d7dac8adb2f46f7e24b61443e47889923dbf6d9bb814f71e87c74db980
+size 4983004016

model-00004-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b83ae6c0acf3c3169aedc480646f69d6fb2b32b60d705f1a030fba1d6862d130
+size 4899035200

model-00005-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:439aa5ac8fe1386c720b4679c09b8409e6e6d36a01c89474985810b0b7f632e5
+size 4983004016

model-00006-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c1b53c5237a8ad3b6a8e02c9628813bee9304bc1b59c6f6abfa54630890d09
+size 4983004016

model-00007-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d4d1732d1567f6145187232f97350e3a7cfadce811de314390df23de59581bb
+size 4899035248

model-00008-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb22551b775e6392d2ec122f3f67f9b83e69bc6053440d19d9f05de77feee291
+size 4983004072

model-00009-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7795b45e140812990ba22294da5dc67e70dd19d08e8837d1b80cd11f435f7766
+size 4983004072

model-00010-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c6f51fe1f458367e4bd43af45e8c0b8f1d61363a59f489e213b791997d4655a
+size 4899035248

model-00011-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:195b9c02f0a40bbdb26cc65d497b6a3bce1dfd0d6a261553bdd33ca8ecf06d81
+size 4983004072

model-00012-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a47a64a5f1e4ade450e89a3e8cf74bee5534b4c4d1be81bf169bd06a5dfb6194
+size 4983004072

model-00013-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5166acc27c9279548c3abfef9e0c81ed9ab9a154469214b04ae42c82c6ad1a6a
+size 4983004072

model-00014-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77d76384ac4489714cd787ca917cf5543d48d907f7a1bccb2524b0e358ee4d8f
+size 4899035248

model-00015-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1ac06972e08d877184bd9114bacc30da79277a39f7c75669343be23c7a57431
+size 4983004072

model-00016-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45fc3b810b670286612f8848c67ad7a2d61d96040ab9d7ace5bc93e5cca03ce3
+size 4983004072

model-00017-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d653889395be0e7a42386dcd74b0a0f91d9ec8a3b4c252f6b2e0bfec570991c8
+size 4899035248

model-00018-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84dc7103c6e76257c56a76a1ab4db0b73af98f8c0685545dc9917b4aef694880
+size 4983004072

model-00019-of-00019.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05dcda152d4bbdb7b287843532350a67b1fccc31f0adaaa244f44eb28ec28879
+size 4221679088

model.safetensors.index.json

@@ -0,0 +1,1002 @@
+{
+  "metadata": {
+    "total_size": 93405585408
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00019-of-00019.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00019.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.block_sparse_moe.gate.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00019.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00019.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.block_sparse_moe.gate.weight": "model-00006-of-00019.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00007-of-00019.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00006-of-00019.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00006-of-00019.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00006-of-00019.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00006-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w3.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w1.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w2.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.11.block_sparse_moe.gate.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00008-of-00019.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00008-of-00019.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00007-of-00019.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00007-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.block_sparse_moe.gate.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00008-of-00019.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w2.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w3.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w1.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.block_sparse_moe.gate.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00009-of-00019.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00008-of-00019.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00008-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.block_sparse_moe.gate.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00009-of-00019.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.block_sparse_moe.gate.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00010-of-00019.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00009-of-00019.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00009-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w3.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w1.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w2.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.16.block_sparse_moe.gate.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00011-of-00019.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00011-of-00019.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00010-of-00019.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00010-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.block_sparse_moe.gate.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00011-of-00019.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w2.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w3.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w1.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.block_sparse_moe.gate.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00012-of-00019.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00011-of-00019.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00011-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.block_sparse_moe.gate.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00012-of-00019.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00012-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.block_sparse_moe.gate.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00002-of-00019.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00002-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.block_sparse_moe.gate.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00013-of-00019.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00012-of-00019.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00012-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w3.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.block_sparse_moe.gate.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00013-of-00019.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00013-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00013-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00013-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.block_sparse_moe.gate.weight": "model-00013-of-00019.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00014-of-00019.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00013-of-00019.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00013-of-00019.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00013-of-00019.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00013-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w2.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w3.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w1.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.23.block_sparse_moe.gate.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00015-of-00019.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00015-of-00019.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00014-of-00019.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00014-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.6.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.6.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.6.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.7.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.7.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.experts.7.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.block_sparse_moe.gate.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00015-of-00019.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w1.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w2.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w3.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.6.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.6.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.6.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.7.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.7.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.experts.7.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.block_sparse_moe.gate.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00016-of-00019.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00015-of-00019.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00015-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.6.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.6.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.6.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.7.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.7.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.experts.7.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.block_sparse_moe.gate.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00016-of-00019.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.6.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.6.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.6.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.7.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.7.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.experts.7.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.block_sparse_moe.gate.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00017-of-00019.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00016-of-00019.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00016-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.6.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.6.w2.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.6.w3.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.7.w1.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.7.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.experts.7.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.28.block_sparse_moe.gate.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00018-of-00019.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00018-of-00019.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00017-of-00019.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00017-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.6.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.6.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.6.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.7.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.7.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.experts.7.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.block_sparse_moe.gate.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00018-of-00019.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00018-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.block_sparse_moe.gate.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00003-of-00019.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00019.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w1.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w2.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w3.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.6.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.6.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.6.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.7.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.7.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.experts.7.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.block_sparse_moe.gate.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00019-of-00019.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00018-of-00019.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00018-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.6.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.6.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.6.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.7.w1.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.7.w2.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.experts.7.w3.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.block_sparse_moe.gate.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00019-of-00019.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00019-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.block_sparse_moe.gate.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00003-of-00019.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00003-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.block_sparse_moe.gate.weight": "model-00003-of-00019.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00004-of-00019.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00003-of-00019.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00003-of-00019.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00003-of-00019.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00003-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.block_sparse_moe.gate.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00005-of-00019.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00004-of-00019.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00004-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.block_sparse_moe.gate.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00005-of-00019.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.block_sparse_moe.gate.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00006-of-00019.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00005-of-00019.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00005-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.block_sparse_moe.gate.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00006-of-00019.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00006-of-00019.safetensors",
+    "model.norm.weight": "model-00019-of-00019.safetensors"
+  }
+}

pytorch_model-00001-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9db548a1dd0eedcedb76d8bcac03a96056e5d1db6491f3d92f225b71c39663c
+size 4892821288

pytorch_model-00002-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb12628856c11311ef213d24e99ed28c280d803a4e857b812c8afdbea556ba86
+size 4983016612

pytorch_model-00003-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8805384ad3c280c920246448560c8b9007469372fd1244337d86e0f9c26f7ba4
+size 4983016696

pytorch_model-00004-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:496e05240e7f8b9c41154f38136ed95e98f03b822627b3f1ec613b5ca1500d73
+size 4899046246

pytorch_model-00005-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0041df495059f1a414053a958acdf796fa1d6046769980938569aad81f6499d
+size 4983016648

pytorch_model-00006-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68edabbb261d6638b71340d74e5337498c5ada320f108f7885f62b5bdd445a4c
+size 4983016612

pytorch_model-00007-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f254ac21ed954dbe4597ecf0e9109d72648258ae9499e78efb45ed042e11605
+size 4899046246

pytorch_model-00008-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4359503716fa958ecc6bfdd708050b1d6990fa4d12bf483faaeb6af11f930367
+size 4983016736

pytorch_model-00009-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ef3ada271f064f30e9b5fd17d9c3c0c54aa431a9efe0c5419f4c9e50283feb0
+size 4983016676

pytorch_model-00010-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0917a1c80e8bd9792a9ee03badb3db9f8d4b0e9f9ff2ef460e5dee25280a826a
+size 4899046246

pytorch_model-00011-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf30083221017ad45734c34d8c29efff235d5e401aa821be1429a3260333e575
+size 4983016700

pytorch_model-00012-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54f62f96c875c0acee39bd20b54ccaff25b9d6227b36b5bc90b9e0630b5cb70d
+size 4983016700

pytorch_model-00013-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f59fb47e0e87c16e0bdb1beec938bc04d7c29810d0aacb8a67223ff6b33b7481
+size 4983016676

pytorch_model-00014-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:107b133a8f5ebd572ab306f2665c2988c39497b0712e925b3f64216ade917e3a
+size 4899046246

pytorch_model-00015-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:388ee9fef1b7f02203c009c13f8dccce636b8607f6c5249a7b001de74c6607fe
+size 4983016736

pytorch_model-00016-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c407549e29b0c464d0813f8a8c26ccd0a54e4115883f7ce87736d26607f32f
+size 4983016676

pytorch_model-00017-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbecd67a459ba573c5dfdb9c11515dbe2016ab2435d5b0a3c3388d07df2a4c9b
+size 4899046246

pytorch_model-00018-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9354fc3cc054c0599b8e1ec074c971ff63938d2bd64f5dfa5c2fb4b338c19d38
+size 4983016712

pytorch_model-00019-of-00019.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64b6a59251d0c8425112a820374ea9e81d414f8dcd3830cbebdff381136240ce
+size 4221689154

pytorch_model.bin.index.json

@@ -0,0 +1,1002 @@
+{
+  "metadata": {
+    "total_size": 93405585408
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00019-of-00019.bin",
+    "model.embed_tokens.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.block_sparse_moe.gate.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.block_sparse_moe.gate.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.input_layernorm.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.block_sparse_moe.gate.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.10.input_layernorm.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.11.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.11.block_sparse_moe.gate.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.input_layernorm.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00007-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.block_sparse_moe.gate.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.input_layernorm.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.block_sparse_moe.gate.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.input_layernorm.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00008-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.block_sparse_moe.gate.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.input_layernorm.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.block_sparse_moe.gate.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.input_layernorm.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00009-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.16.block_sparse_moe.gate.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.input_layernorm.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00010-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.block_sparse_moe.gate.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.input_layernorm.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.block_sparse_moe.gate.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.input_layernorm.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00011-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.block_sparse_moe.gate.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.input_layernorm.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.block_sparse_moe.gate.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.input_layernorm.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.block_sparse_moe.gate.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.input_layernorm.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00012-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.block_sparse_moe.gate.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.input_layernorm.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.block_sparse_moe.gate.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.22.input_layernorm.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00013-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.23.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.23.block_sparse_moe.gate.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.input_layernorm.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00014-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.block_sparse_moe.gate.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.input_layernorm.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.block_sparse_moe.gate.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.input_layernorm.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00015-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.block_sparse_moe.gate.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.input_layernorm.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.block_sparse_moe.gate.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.input_layernorm.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00016-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.28.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.28.block_sparse_moe.gate.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.input_layernorm.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00017-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.block_sparse_moe.gate.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.input_layernorm.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.block_sparse_moe.gate.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.input_layernorm.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00002-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.block_sparse_moe.gate.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.input_layernorm.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00018-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.block_sparse_moe.gate.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.input_layernorm.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00019-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.block_sparse_moe.gate.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.input_layernorm.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.block_sparse_moe.gate.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.5.input_layernorm.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00003-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.block_sparse_moe.gate.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.input_layernorm.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00004-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.block_sparse_moe.gate.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.input_layernorm.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.block_sparse_moe.gate.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.input_layernorm.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00005-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.0.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.0.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.0.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.1.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.1.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.1.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.2.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.2.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.2.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.3.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.3.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.3.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.4.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.4.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.4.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.5.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.5.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.5.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.6.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.6.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.6.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.7.w1.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.7.w2.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.experts.7.w3.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.block_sparse_moe.gate.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.input_layernorm.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00006-of-00019.bin",
+    "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00006-of-00019.bin",
+    "model.norm.weight": "pytorch_model-00019-of-00019.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f70867dd9fe490004d9bb8bd993b009ecb03b8dd015c8877e8c2a44db409b919
+size 5112