Hugging Face's logo

TroyDoesAI
/
Mermaid-Dolphin-Mixtral-2x7b

Text Generation
Transformers
Safetensors
mixtral
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
TroyDoesAI commited on
Commit
79a9a23
·
verified ·
1 Parent(s): d9ae546

Dolphin Mermaid 2x7b Merge

Browse files
Files changed (10) hide show
  1. config.json +6 -6
  2. mergekit_moe_config.yml +154 -0
  3. model-00001-of-00003.safetensors +3 -0
  4. model-00002-of-00003.safetensors +3 -0
  5. model-00003-of-00003.safetensors +3 -0
  6. model.safetensors.index.json +1 -426
  7. special_tokens_map.json +7 -8
  8. tokenizer.json +1 -26
  9. tokenizer.model +3 -0
  10. tokenizer_config.json +10 -25
config.json CHANGED
@@ -1,11 +1,11 @@
1
  {
2
- "_name_or_path": ".\\Mermaid\\",
3
  "architectures": [
4
  "MixtralForCausalLM"
5
  ],
6
  "attention_dropout": 0.0,
7
  "bos_token_id": 1,
8
- "eos_token_id": 32000,
9
  "hidden_act": "silu",
10
  "hidden_size": 4096,
11
  "initializer_range": 0.02,
@@ -19,12 +19,12 @@
19
  "num_local_experts": 2,
20
  "output_router_logits": false,
21
  "rms_norm_eps": 1e-05,
22
- "rope_theta": 1000000.0,
23
  "router_aux_loss_coef": 0.001,
24
  "sliding_window": null,
25
  "tie_word_embeddings": false,
26
  "torch_dtype": "float16",
27
- "transformers_version": "4.37.2",
28
- "use_cache": false,
29
- "vocab_size": 32002
30
  }
 
1
  {
2
+ "_name_or_path": "TroyDoesAI/MermaidMistral",
3
  "architectures": [
4
  "MixtralForCausalLM"
5
  ],
6
  "attention_dropout": 0.0,
7
  "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
  "hidden_act": "silu",
10
  "hidden_size": 4096,
11
  "initializer_range": 0.02,
 
19
  "num_local_experts": 2,
20
  "output_router_logits": false,
21
  "rms_norm_eps": 1e-05,
22
+ "rope_theta": 10000.0,
23
  "router_aux_loss_coef": 0.001,
24
  "sliding_window": null,
25
  "tie_word_embeddings": false,
26
  "torch_dtype": "float16",
27
+ "transformers_version": "4.38.1",
28
+ "use_cache": true,
29
+ "vocab_size": 32000
30
  }
mergekit_moe_config.yml ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ base_model: TroyDoesAI/MermaidMistral
2
+ gate_mode: hidden # one of "hidden", "cheap_embed", or "random"
3
+
4
+ # There are three methods for populating the MoE gates implemented.
5
+
6
+ # "hidden"
7
+ # Uses the hidden state representations of the positive/negative prompts for MoE gate parameters. Best quality and most effective option; the default. Requires evaluating each prompt using the base model so you might not be able to use this on constrained hardware (depending on the model). You can use --load-in-8bit or --load-in-4bit to reduce VRAM usage.
8
+
9
+ # "cheap_embed"
10
+ # Uses only the raw token embedding of the prompts, using the same gate parameters for every layer. Distinctly less effective than "hidden". Can be run on much, much lower end hardware.
11
+
12
+ # "random"
13
+ # Randomly initializes the MoE gates. Good for if you are going to fine tune the model afterwards, or maybe if you want something a little unhinged? I won't judge.
14
+
15
+ experts:
16
+ - source_model: cognitivecomputations/dolphin-2.8-mistral-7b-v02
17
+ positive_prompts:
18
+ - "Assist me in creating a mermaid diagram"
19
+ - "Assist me in creating a mermaid diagram that shows"
20
+ - "Assistant"
21
+ - "Help me create a mermaid diagram"
22
+ - "Help me create a mermaid diagram that shows"
23
+ - "Help me create a mermaid diagram that illustrates"
24
+ - "Help me create a mermaid diagram that demonstrates"
25
+ - "Help me create a mermaid diagram that explains"
26
+ - "Help me create a mermaid diagram that describes"
27
+ - "Help me create a mermaid diagram that visualizes"
28
+ - "Help me create a mermaid diagram that represents"
29
+ - "Update the mermaid diagram"
30
+ - "Update a mermaid diagram"
31
+ - "Update the mermaid diagram that shows"
32
+ - "Update a mermaid diagram that shows"
33
+ - "Update the mermaid diagram that illustrates"
34
+ - "### Instruction"
35
+ - "instruction"
36
+ - "### Input"
37
+ - "input"
38
+ - "### Response"
39
+ - "response"
40
+ - "code"
41
+ - "python"
42
+ - "javascript"
43
+ - "programming"
44
+ - "algorithm"
45
+ - "diagram"
46
+ - "flowchart"
47
+ - "sequence"
48
+ - "uml"
49
+ - "mermaid js syntax"
50
+ - "flow"
51
+ - "!mermaid"
52
+ - "!KnowledgeGraph"
53
+ - "mermaid"
54
+ - "explain"
55
+ - "describe"
56
+ - "draw"
57
+ - "create"
58
+ - "Create the mermaid diagram"
59
+ - "Create a mermaid diagram"
60
+ - "Create a mermaid diagram that shows"
61
+ - "Create a mermaid diagram that illustrates"
62
+ - "Create a mermaid diagram that demonstrates"
63
+ - "Create a mermaid diagram that explains"
64
+ - "Create a mermaid diagram that describes"
65
+ - "Create a mermaid diagram that visualizes"
66
+ - "Create a mermaid diagram that represents"
67
+ - "Create a mermaid diagram that displays"
68
+ - "Create a mermaid diagram that depicts"
69
+ - "Create a mermaid diagram that outlines"
70
+ - "Create a mermaid diagram that details"
71
+ - "Create a mermaid diagram that exhibits"
72
+ - "Create a mermaid diagram that showcases"
73
+ - "Create a mermaid diagram that manifests"
74
+ - "Create a mermaid diagram that embodies"
75
+ - "Create a mermaid diagram that symbolizes"
76
+ - "Create a mermaid diagram that signifies"
77
+ - "Create the highly detailed mermaid diagram for the following so that I can look at the diagram and understand every single function as its own flow diagram using subgraphs of finely detailed step by step flow."
78
+ - "Create a mermaid diagram that shows the flow of the following code"
79
+ - "Create the mermaid diagram for the following code:"
80
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner"
81
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs"
82
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs and detailed step by step flow"
83
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs and detailed step by step flow of each function"
84
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions"
85
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions and their subfunctions"
86
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions and their subfunctions and their subfunctions"
87
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions and their subfunctions and their subfunctions and their subfunctions"
88
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions and their subfunctions and their subfunctions and their subfunctions and their subfunctions"
89
+ - "Create a mermaid diagram that shows the flow of the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions and their subfunctions and their subfunctions and their subfunctions and their subfunctions and their subfunctions"
90
+ - "Can you create the mermaid diagram to show the change between"
91
+ - "Can you create the mermaid diagram to show the difference between"
92
+ - "Can you create the mermaid diagram to show the transition between"
93
+ - "Can you create the mermaid diagram to show the transformation between"
94
+ - "Can you create the mermaid diagram to show the conversion between"
95
+ - "Create the mermaid diagram for the following code:"
96
+ - "Create the mermaid diagram for the following code"
97
+ - "Create the mermaid diagram for the following code in a detailed manner"
98
+ - "Create the mermaid diagram for the following code in a detailed manner with subgraphs"
99
+ - "Create the mermaid diagram for the following code in a detailed manner with subgraphs and detailed step by step flow"
100
+ - "Create the mermaid diagram for the following code in a detailed manner with subgraphs and detailed step by step flow of each function"
101
+ - "Create the mermaid diagram for the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions"
102
+ - "Create the mermaid diagram for the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions and their subfunctions"
103
+ - "Create the mermaid diagram for the following code in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions and their subfunctions and their subfunctions"
104
+ - "Create the mermaid diagram for the following story:"
105
+ - "Create the mermaid diagram for the following story"
106
+ - "Create the mermaid diagram for the following story in a detailed manner"
107
+ - "Create the mermaid diagram for the following story in a detailed manner with subgraphs"
108
+ - "Create the mermaid diagram for the following story in a detailed manner with subgraphs and detailed step by step flow"
109
+ - "Create the mermaid diagram for the following story in a detailed manner with subgraphs and detailed step by step flow of each function"
110
+ - "Create the mermaid diagram for the following story in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions"
111
+ - "Create the mermaid diagram for the following story in a detailed manner with subgraphs and detailed step by step flow of each function and its subfunctions and their subfunctions"
112
+ negative_prompts:
113
+ - "graph TB"
114
+ - "graph TD"
115
+ - "graph LR"
116
+ - "graph RL"
117
+ - "graph TB;"
118
+ - "graph TD;"
119
+ - "graph LR;"
120
+ - "graph RL;"
121
+ - ";"
122
+ - "-->"
123
+ - "["
124
+ - "]"
125
+ - "direction TB"
126
+ - "direction TD"
127
+ - "subgraph"
128
+ - "end"
129
+ - "```"
130
+
131
+
132
+ - source_model: TroyDoesAI/MermaidMistral
133
+ positive_prompts:
134
+ - "graph TB"
135
+ - "graph TD"
136
+ - "graph LR"
137
+ - "graph RL"
138
+ - "graph TB;"
139
+ - "graph TD;"
140
+ - "graph LR;"
141
+ - "graph RL;"
142
+ - ";"
143
+ - "-->"
144
+ - "["
145
+ - "]"
146
+ - "direction TB"
147
+ - "direction TD"
148
+ - "subgraph"
149
+ - "end"
150
+ - "A[Start Main]"
151
+ - "[End Main];"
152
+ - "```"
153
+
154
+
model-00001-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:87c813005c878677b039a388b04f7f72f295ea8f45c048cfb258a756084cddec
3
+ size 9986868936
model-00002-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9a01dacc808ca50a9173e6759706d0b38fb1a28e35c446733d512b1e3f4b571
3
+ size 9898781048
model-00003-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ffc0b9c45f548aaca5e3af3923946bbcf7caef9ac49208f7a270323d67c6a7c
3
+ size 5872679032
model.safetensors.index.json CHANGED
@@ -1,426 +1 @@
1
- {
2
- "metadata": {
3
- "total_size": 25758310400
4
- },
5
- "weight_map": {
6
- "lm_head.weight": "model-00006-of-00006.safetensors",
7
- "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
8
- "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
9
- "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
10
- "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
11
- "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
12
- "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
13
- "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
14
- "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
15
- "model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
16
- "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
17
- "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
18
- "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
19
- "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
20
- "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
21
- "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
22
- "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
23
- "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
24
- "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
25
- "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
26
- "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
27
- "model.layers.1.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
28
- "model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
29
- "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
30
- "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
31
- "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
32
- "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
33
- "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
34
- "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
35
- "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
36
- "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
37
- "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
38
- "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
39
- "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
40
- "model.layers.10.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
41
- "model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors",
42
- "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
43
- "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
44
- "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
45
- "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
46
- "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
47
- "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
48
- "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
49
- "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
50
- "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
51
- "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
52
- "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
53
- "model.layers.11.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
54
- "model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors",
55
- "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
56
- "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
57
- "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
58
- "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
59
- "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
60
- "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
61
- "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
62
- "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
63
- "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
64
- "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
65
- "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
66
- "model.layers.12.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
67
- "model.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors",
68
- "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
69
- "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
70
- "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
71
- "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
72
- "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
73
- "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
74
- "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
75
- "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
76
- "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
77
- "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
78
- "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
79
- "model.layers.13.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
80
- "model.layers.13.input_layernorm.weight": "model-00003-of-00006.safetensors",
81
- "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
82
- "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
83
- "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
84
- "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
85
- "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
86
- "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
87
- "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
88
- "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
89
- "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
90
- "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
91
- "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
92
- "model.layers.14.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
93
- "model.layers.14.input_layernorm.weight": "model-00003-of-00006.safetensors",
94
- "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
95
- "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
96
- "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
97
- "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
98
- "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
99
- "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
100
- "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
101
- "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
102
- "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
103
- "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
104
- "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
105
- "model.layers.15.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
106
- "model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
107
- "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
108
- "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
109
- "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
110
- "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
111
- "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
112
- "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
113
- "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
114
- "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
115
- "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
116
- "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
117
- "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
118
- "model.layers.16.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
119
- "model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors",
120
- "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
121
- "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
122
- "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
123
- "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
124
- "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
125
- "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
126
- "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
127
- "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
128
- "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
129
- "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
130
- "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
131
- "model.layers.17.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
132
- "model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors",
133
- "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
134
- "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
135
- "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
136
- "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
137
- "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
138
- "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
139
- "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
140
- "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
141
- "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
142
- "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
143
- "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
144
- "model.layers.18.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
145
- "model.layers.18.input_layernorm.weight": "model-00004-of-00006.safetensors",
146
- "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
147
- "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
148
- "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
149
- "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
150
- "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
151
- "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
152
- "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
153
- "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
154
- "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
155
- "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
156
- "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
157
- "model.layers.19.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
158
- "model.layers.19.input_layernorm.weight": "model-00004-of-00006.safetensors",
159
- "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
160
- "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
161
- "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
162
- "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
163
- "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
164
- "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
165
- "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
166
- "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
167
- "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
168
- "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
169
- "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
170
- "model.layers.2.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
171
- "model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
172
- "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
173
- "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
174
- "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
175
- "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
176
- "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
177
- "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
178
- "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
179
- "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
180
- "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
181
- "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
182
- "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
183
- "model.layers.20.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
184
- "model.layers.20.input_layernorm.weight": "model-00004-of-00006.safetensors",
185
- "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
186
- "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
187
- "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
188
- "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
189
- "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
190
- "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
191
- "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
192
- "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
193
- "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
194
- "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
195
- "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
196
- "model.layers.21.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
197
- "model.layers.21.input_layernorm.weight": "model-00004-of-00006.safetensors",
198
- "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
199
- "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
200
- "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
201
- "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
202
- "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
203
- "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
204
- "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
205
- "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
206
- "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
207
- "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
208
- "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
209
- "model.layers.22.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
210
- "model.layers.22.input_layernorm.weight": "model-00004-of-00006.safetensors",
211
- "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
212
- "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
213
- "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
214
- "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
215
- "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
216
- "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
217
- "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
218
- "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
219
- "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
220
- "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
221
- "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
222
- "model.layers.23.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
223
- "model.layers.23.input_layernorm.weight": "model-00004-of-00006.safetensors",
224
- "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
225
- "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
226
- "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
227
- "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
228
- "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
229
- "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
230
- "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
231
- "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
232
- "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
233
- "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
234
- "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
235
- "model.layers.24.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
236
- "model.layers.24.input_layernorm.weight": "model-00005-of-00006.safetensors",
237
- "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
238
- "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
239
- "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
240
- "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
241
- "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
242
- "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
243
- "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
244
- "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
245
- "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
246
- "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
247
- "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
248
- "model.layers.25.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
249
- "model.layers.25.input_layernorm.weight": "model-00005-of-00006.safetensors",
250
- "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
251
- "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
252
- "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
253
- "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
254
- "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
255
- "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
256
- "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
257
- "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
258
- "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
259
- "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
260
- "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
261
- "model.layers.26.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
262
- "model.layers.26.input_layernorm.weight": "model-00005-of-00006.safetensors",
263
- "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
264
- "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
265
- "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
266
- "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
267
- "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
268
- "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
269
- "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
270
- "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
271
- "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
272
- "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
273
- "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
274
- "model.layers.27.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
275
- "model.layers.27.input_layernorm.weight": "model-00005-of-00006.safetensors",
276
- "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
277
- "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
278
- "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
279
- "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
280
- "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
281
- "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
282
- "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
283
- "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
284
- "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
285
- "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
286
- "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
287
- "model.layers.28.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
288
- "model.layers.28.input_layernorm.weight": "model-00005-of-00006.safetensors",
289
- "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
290
- "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
291
- "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
292
- "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
293
- "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
294
- "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
295
- "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
296
- "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
297
- "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
298
- "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
299
- "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
300
- "model.layers.29.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
301
- "model.layers.29.input_layernorm.weight": "model-00005-of-00006.safetensors",
302
- "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
303
- "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
304
- "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
305
- "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
306
- "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
307
- "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
308
- "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
309
- "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
310
- "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
311
- "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
312
- "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
313
- "model.layers.3.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
314
- "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
315
- "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
316
- "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
317
- "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
318
- "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
319
- "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
320
- "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
321
- "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
322
- "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
323
- "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
324
- "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
325
- "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
326
- "model.layers.30.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
327
- "model.layers.30.input_layernorm.weight": "model-00005-of-00006.safetensors",
328
- "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
329
- "model.layers.30.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
330
- "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
331
- "model.layers.30.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
332
- "model.layers.30.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
333
- "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00006.safetensors",
334
- "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00006.safetensors",
335
- "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00006.safetensors",
336
- "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00006.safetensors",
337
- "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00006.safetensors",
338
- "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00006.safetensors",
339
- "model.layers.31.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
340
- "model.layers.31.input_layernorm.weight": "model-00006-of-00006.safetensors",
341
- "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
342
- "model.layers.31.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
343
- "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
344
- "model.layers.31.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
345
- "model.layers.31.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
346
- "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
347
- "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
348
- "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
349
- "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
350
- "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
351
- "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
352
- "model.layers.4.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
353
- "model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
354
- "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
355
- "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
356
- "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
357
- "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
358
- "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
359
- "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
360
- "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
361
- "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
362
- "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
363
- "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
364
- "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
365
- "model.layers.5.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
366
- "model.layers.5.input_layernorm.weight": "model-00001-of-00006.safetensors",
367
- "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
368
- "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
369
- "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
370
- "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
371
- "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
372
- "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
373
- "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
374
- "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
375
- "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
376
- "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
377
- "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
378
- "model.layers.6.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
379
- "model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
380
- "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
381
- "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
382
- "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
383
- "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
384
- "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
385
- "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
386
- "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
387
- "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
388
- "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
389
- "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
390
- "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
391
- "model.layers.7.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
392
- "model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
393
- "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
394
- "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
395
- "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
396
- "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
397
- "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
398
- "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
399
- "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
400
- "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
401
- "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
402
- "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
403
- "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
404
- "model.layers.8.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
405
- "model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors",
406
- "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
407
- "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
408
- "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
409
- "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
410
- "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
411
- "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
412
- "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
413
- "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
414
- "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
415
- "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
416
- "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
417
- "model.layers.9.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
418
- "model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
419
- "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
420
- "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
421
- "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
422
- "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
423
- "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
424
- "model.norm.weight": "model-00006-of-00006.safetensors"
425
- }
426
- }
 
1
+ {"metadata": {"mergekit_version": "0.0.4.1", "total_size": 25758277632}, "weight_map": {"model.embed_tokens.weight": "model-00001-of-00003.safetensors", "model.norm.weight": "model-00001-of-00003.safetensors", "lm_head.weight": "model-00001-of-00003.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.0.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.1.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.2.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.3.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.4.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.5.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.6.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.7.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.8.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.9.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.10.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.11.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.12.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.13.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.14.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.15.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.16.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.17.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.18.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.19.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.20.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.21.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.22.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.23.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.24.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors"}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
special_tokens_map.json CHANGED
@@ -1,4 +1,9 @@
1
  {
 
 
 
 
 
2
  "bos_token": {
3
  "content": "<s>",
4
  "lstrip": false,
@@ -7,19 +12,13 @@
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "<|im_end|>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": {
17
- "content": "<s>",
18
  "lstrip": false,
19
  "normalized": false,
20
  "rstrip": false,
21
  "single_word": false
22
  },
 
23
  "unk_token": {
24
  "content": "<unk>",
25
  "lstrip": false,
 
1
  {
2
+ "additional_special_tokens": [
3
+ "<unk>",
4
+ "<s>",
5
+ "</s>"
6
+ ],
7
  "bos_token": {
8
  "content": "<s>",
9
  "lstrip": false,
 
12
  "single_word": false
13
  },
14
  "eos_token": {
15
+ "content": "</s>",
 
 
 
 
 
 
 
16
  "lstrip": false,
17
  "normalized": false,
18
  "rstrip": false,
19
  "single_word": false
20
  },
21
+ "pad_token": "<s>",
22
  "unk_token": {
23
  "content": "<unk>",
24
  "lstrip": false,
tokenizer.json CHANGED
@@ -1,14 +1,7 @@
1
  {
2
  "version": "1.0",
3
  "truncation": null,
4
- "padding": {
5
- "strategy": "BatchLongest",
6
- "direction": "Left",
7
- "pad_to_multiple_of": null,
8
- "pad_id": 1,
9
- "pad_type_id": 0,
10
- "pad_token": "<s>"
11
- },
12
  "added_tokens": [
13
  {
14
  "id": 0,
@@ -36,24 +29,6 @@
36
  "rstrip": false,
37
  "normalized": false,
38
  "special": true
39
- },
40
- {
41
- "id": 32000,
42
- "content": "<|im_end|>",
43
- "single_word": false,
44
- "lstrip": false,
45
- "rstrip": false,
46
- "normalized": false,
47
- "special": true
48
- },
49
- {
50
- "id": 32001,
51
- "content": "<|im_start|>",
52
- "single_word": false,
53
- "lstrip": false,
54
- "rstrip": false,
55
- "normalized": false,
56
- "special": false
57
  }
58
  ],
59
  "normalizer": {
 
1
  {
2
  "version": "1.0",
3
  "truncation": null,
4
+ "padding": null,
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
 
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  }
33
  ],
34
  "normalizer": {
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json CHANGED
@@ -1,7 +1,6 @@
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
4
- "add_prefix_space": true,
5
  "added_tokens_decoder": {
6
  "0": {
7
  "content": "<unk>",
@@ -26,39 +25,25 @@
26
  "rstrip": false,
27
  "single_word": false,
28
  "special": true
29
- },
30
- "32000": {
31
- "content": "<|im_end|>",
32
- "lstrip": false,
33
- "normalized": false,
34
- "rstrip": false,
35
- "single_word": false,
36
- "special": true
37
- },
38
- "32001": {
39
- "content": "<|im_start|>",
40
- "lstrip": false,
41
- "normalized": false,
42
- "rstrip": false,
43
- "single_word": false,
44
- "special": false
45
  }
46
  },
 
 
 
 
 
47
  "bos_token": "<s>",
48
- "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
49
  "clean_up_tokenization_spaces": false,
50
- "eos_token": "<|im_end|>",
51
  "legacy": true,
52
- "max_length": null,
53
- "model_max_length": 1000000000000000019884624838656,
54
- "pad_to_multiple_of": null,
55
  "pad_token": "<s>",
56
- "pad_token_type_id": 0,
57
  "padding_side": "left",
58
  "sp_model_kwargs": {},
59
  "spaces_between_special_tokens": false,
 
60
  "tokenizer_class": "LlamaTokenizer",
61
  "unk_token": "<unk>",
62
- "use_default_system_prompt": false,
63
- "use_fast": true
64
  }
 
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
 
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
 
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  }
29
  },
30
+ "additional_special_tokens": [
31
+ "<unk>",
32
+ "<s>",
33
+ "</s>"
34
+ ],
35
  "bos_token": "<s>",
36
+ "chat_template": "{% for message in messages %}{{bos_token + message['role'] + '\n' + message['content'] + eos_token + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\n' }}{% endif %}",
37
  "clean_up_tokenization_spaces": false,
38
+ "eos_token": "</s>",
39
  "legacy": true,
40
+ "model_max_length": 8192,
 
 
41
  "pad_token": "<s>",
 
42
  "padding_side": "left",
43
  "sp_model_kwargs": {},
44
  "spaces_between_special_tokens": false,
45
+ "split_special_tokens": false,
46
  "tokenizer_class": "LlamaTokenizer",
47
  "unk_token": "<unk>",
48
+ "use_default_system_prompt": true
 
49
  }