sparsity_report_Meta-Llama-3.1-8B-wanda-unstructured-0.1.csv

row,layer_id,short_id,layer_type,param_type,shape,nparam,nnz,sparsity,tile_shape,n_tile,n_tile_total,tile_avg,tile_min,tile_med,tile_max,col_avg,col_min,col_med,col_max,row_avg,row_min,row_med,row_max
0,model.layers.0.self_attn.q_proj,tx.0.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0,0.09130859375,0.33984375,0.099853515625,0.0,0.0,1.0,0.099853515625,0.0,0.0625,0.5
1,model.layers.0.self_attn.k_proj,tx.0.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.00048828125,0.09033203125,0.34716796875,0.099853515625,0.0,0.0,1.0,0.099853515625,0.0,0.0625,0.5625
2,model.layers.0.self_attn.v_proj,tx.0.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.00244140625,0.08837890625,0.31884765625,0.099853515625,0.0,0.0,1.0,0.099853515625,0.0,0.0625,0.5
3,model.layers.0.self_attn.o_proj,tx.0.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0048828125,0.06494140625,0.4658203125,0.099853515625,0.0,0.0625,0.6484375,0.099853515625,0.0,0.0625,0.875
4,model.layers.0.mlp.gate_proj,tx.0.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0693359375,0.09619140625,0.22705078125,0.0998535230755806,0.0,0.09375,1.0,0.0998535230755806,0.0,0.0625,0.6875
5,model.layers.0.mlp.up_proj,tx.0.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.068359375,0.09619140625,0.2236328125,0.0998535230755806,0.0,0.09375,1.0,0.0998535230755806,0.0,0.0625,0.6875
6,model.layers.0.mlp.down_proj,tx.0.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.060546875,0.10009765625,0.1416015625,0.0999581515789032,0.0,0.1015625,0.3125,0.0999581515789032,0.0,0.0625,0.625
7,model.layers.1.self_attn.q_proj,tx.1.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0458984375,0.09375,0.2919921875,0.099853515625,0.0,0.09375,1.0,0.099853515625,0.0,0.0625,0.625
8,model.layers.1.self_attn.k_proj,tx.1.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0517578125,0.09423828125,0.271484375,0.099853515625,0.0,0.09375,1.0,0.099853515625,0.0,0.0625,0.5
9,model.layers.1.self_attn.v_proj,tx.1.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.05419921875,0.09326171875,0.2744140625,0.099853515625,0.0,0.09375,1.0,0.099853515625,0.0,0.0625,0.5625
10,model.layers.1.self_attn.o_proj,tx.1.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03076171875,0.0859375,0.341796875,0.099853515625,0.0,0.078125,0.59375,0.099853515625,0.0,0.0625,0.75
11,model.layers.1.mlp.gate_proj,tx.1.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0693359375,0.0966796875,0.2236328125,0.0998535230755806,0.0,0.09375,1.0,0.0998535230755806,0.0,0.0625,0.625
12,model.layers.1.mlp.up_proj,tx.1.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0712890625,0.0966796875,0.22216796875,0.0998535230755806,0.0,0.09375,1.0,0.0998535230755806,0.0,0.0625,0.625
13,model.layers.1.mlp.down_proj,tx.1.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.064453125,0.099609375,0.13525390625,0.0999581515789032,0.0,0.1015625,0.2734375,0.0999581515789032,0.0,0.0625,0.625
14,model.layers.2.self_attn.q_proj,tx.2.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.05908203125,0.0986328125,0.189453125,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
15,model.layers.2.self_attn.k_proj,tx.2.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0615234375,0.0986328125,0.18115234375,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
16,model.layers.2.self_attn.v_proj,tx.2.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07275390625,0.09814453125,0.1787109375,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5
17,model.layers.2.self_attn.o_proj,tx.2.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03564453125,0.08349609375,0.3173828125,0.099853515625,0.0,0.0859375,0.4765625,0.099853515625,0.0,0.0625,0.75
18,model.layers.2.mlp.gate_proj,tx.2.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.0966796875,0.22412109375,0.0998535230755806,0.0,0.09375,1.0,0.0998535230755806,0.0,0.0625,0.625
19,model.layers.2.mlp.up_proj,tx.2.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.06982421875,0.0966796875,0.22802734375,0.0998535230755806,0.0,0.09375,1.0,0.0998535230755806,0.0,0.0625,0.625
20,model.layers.2.mlp.down_proj,tx.2.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0654296875,0.10009765625,0.14453125,0.0999581515789032,0.0,0.1015625,0.2890625,0.0999581515789032,0.0,0.0625,0.6875
21,model.layers.3.self_attn.q_proj,tx.3.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.068359375,0.099609375,0.18212890625,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
22,model.layers.3.self_attn.k_proj,tx.3.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.06982421875,0.099609375,0.16650390625,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.625
23,model.layers.3.self_attn.v_proj,tx.3.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.072265625,0.09912109375,0.17919921875,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.625
24,model.layers.3.self_attn.o_proj,tx.3.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.044921875,0.09521484375,0.18603515625,0.099853515625,0.0,0.09375,0.328125,0.099853515625,0.0,0.0625,0.625
25,model.layers.3.mlp.gate_proj,tx.3.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0693359375,0.09765625,0.22412109375,0.0998535230755806,0.0,0.09375,1.0,0.0998535230755806,0.0,0.0625,0.5625
26,model.layers.3.mlp.up_proj,tx.3.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07275390625,0.09765625,0.2236328125,0.0998535230755806,0.0,0.09375,1.0,0.0998535230755806,0.0,0.0625,0.5625
27,model.layers.3.mlp.down_proj,tx.3.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06884765625,0.099609375,0.140625,0.0999581515789032,0.0,0.1015625,0.2734375,0.0999581515789032,0.0,0.0625,0.6875
28,model.layers.4.self_attn.q_proj,tx.4.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06640625,0.099609375,0.169921875,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
29,model.layers.4.self_attn.k_proj,tx.4.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0673828125,0.10009765625,0.1591796875,0.099853515625,0.0,0.1015625,0.9765625,0.099853515625,0.0,0.0625,0.5625
30,model.layers.4.self_attn.v_proj,tx.4.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07080078125,0.09912109375,0.17138671875,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
31,model.layers.4.self_attn.o_proj,tx.4.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.037109375,0.08642578125,0.28857421875,0.099853515625,0.0,0.0859375,0.453125,0.099853515625,0.0,0.0625,0.8125
32,model.layers.4.mlp.gate_proj,tx.4.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.09912109375,0.173828125,0.0998535230755806,0.0,0.1015625,1.0,0.0998535230755806,0.0,0.0625,0.625
33,model.layers.4.mlp.up_proj,tx.4.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07177734375,0.09912109375,0.171875,0.0998535230755806,0.0,0.1015625,1.0,0.0998535230755806,0.0,0.0625,0.5625
34,model.layers.4.mlp.down_proj,tx.4.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06982421875,0.10009765625,0.1337890625,0.0999581515789032,0.0,0.1015625,0.421875,0.0999581515789032,0.0,0.0625,0.625
35,model.layers.5.self_attn.q_proj,tx.5.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06396484375,0.099609375,0.13330078125,0.099853515625,0.0,0.1015625,0.28125,0.099853515625,0.0,0.0625,0.5625
36,model.layers.5.self_attn.k_proj,tx.5.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.06005859375,0.10009765625,0.14013671875,0.099853515625,0.0,0.1015625,0.4375,0.099853515625,0.0,0.0625,0.5
37,model.layers.5.self_attn.v_proj,tx.5.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0712890625,0.09912109375,0.17724609375,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5
38,model.layers.5.self_attn.o_proj,tx.5.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.02783203125,0.08056640625,0.25341796875,0.099853515625,0.0,0.0859375,0.4375,0.099853515625,0.0,0.0625,0.6875
39,model.layers.5.mlp.gate_proj,tx.5.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.06982421875,0.099609375,0.16748046875,0.0998535230755806,0.0,0.1015625,1.0,0.0998535230755806,0.0,0.0625,0.625
40,model.layers.5.mlp.up_proj,tx.5.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07421875,0.099609375,0.173828125,0.0998535230755806,0.0,0.1015625,1.0,0.0998535230755806,0.0,0.0625,0.625
41,model.layers.5.mlp.down_proj,tx.5.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.07080078125,0.10009765625,0.134765625,0.0999581515789032,0.0,0.1015625,0.296875,0.0999581515789032,0.0,0.0625,0.625
42,model.layers.6.self_attn.q_proj,tx.6.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06787109375,0.099609375,0.13427734375,0.099853515625,0.0,0.1015625,0.2578125,0.099853515625,0.0,0.0625,0.5625
43,model.layers.6.self_attn.k_proj,tx.6.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.06982421875,0.099609375,0.1318359375,0.099853515625,0.0,0.1015625,0.3984375,0.099853515625,0.0,0.0625,0.5625
44,model.layers.6.self_attn.v_proj,tx.6.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07421875,0.099609375,0.1640625,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5
45,model.layers.6.self_attn.o_proj,tx.6.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03759765625,0.0927734375,0.2080078125,0.099853515625,0.0,0.09375,0.359375,0.099853515625,0.0,0.0625,0.6875
46,model.layers.6.mlp.gate_proj,tx.6.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0693359375,0.099609375,0.17529296875,0.0998535230755806,0.0,0.1015625,1.0,0.0998535230755806,0.0,0.0625,0.5625
47,model.layers.6.mlp.up_proj,tx.6.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07080078125,0.099609375,0.17431640625,0.0998535230755806,0.0,0.1015625,1.0,0.0998535230755806,0.0,0.0625,0.625
48,model.layers.6.mlp.down_proj,tx.6.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.07177734375,0.099609375,0.13818359375,0.0999581515789032,0.0,0.1015625,0.3671875,0.0999581515789032,0.0,0.0625,0.5625
49,model.layers.7.self_attn.q_proj,tx.7.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.064453125,0.099609375,0.12939453125,0.099853515625,0.0,0.1015625,0.2734375,0.099853515625,0.0,0.0625,0.5625
50,model.layers.7.self_attn.k_proj,tx.7.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07080078125,0.10009765625,0.1318359375,0.099853515625,0.0,0.1015625,0.4296875,0.099853515625,0.0,0.0625,0.5
51,model.layers.7.self_attn.v_proj,tx.7.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07080078125,0.09912109375,0.171875,0.099853515625,0.0,0.1015625,0.9765625,0.099853515625,0.0,0.0625,0.5625
52,model.layers.7.self_attn.o_proj,tx.7.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.05712890625,0.09814453125,0.16162109375,0.099853515625,0.0,0.1015625,0.3203125,0.099853515625,0.0,0.0625,0.5625
53,model.layers.7.mlp.gate_proj,tx.7.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.099609375,0.13232421875,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.625
54,model.layers.7.mlp.up_proj,tx.7.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07421875,0.099609375,0.1298828125,0.0998535230755806,0.0,0.1015625,0.2421875,0.0998535230755806,0.0,0.0625,0.6875
55,model.layers.7.mlp.down_proj,tx.7.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06787109375,0.099609375,0.13525390625,0.0999581515789032,0.0,0.1015625,0.3203125,0.0999581515789032,0.0,0.0625,0.625
56,model.layers.8.self_attn.q_proj,tx.8.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0693359375,0.099609375,0.13330078125,0.099853515625,0.0,0.1015625,0.2734375,0.099853515625,0.0,0.0625,0.625
57,model.layers.8.self_attn.k_proj,tx.8.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.072265625,0.099609375,0.13037109375,0.099853515625,0.0,0.1015625,0.328125,0.099853515625,0.0,0.0625,0.5625
58,model.layers.8.self_attn.v_proj,tx.8.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0712890625,0.099609375,0.17431640625,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5625
59,model.layers.8.self_attn.o_proj,tx.8.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.04833984375,0.09619140625,0.19091796875,0.099853515625,0.0,0.09375,0.3515625,0.099853515625,0.0,0.0625,0.625
60,model.layers.8.mlp.gate_proj,tx.8.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07177734375,0.099609375,0.17626953125,0.0998535230755806,0.0,0.1015625,1.0,0.0998535230755806,0.0,0.0625,0.625
61,model.layers.8.mlp.up_proj,tx.8.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07275390625,0.099609375,0.17529296875,0.0998535230755806,0.0,0.1015625,1.0,0.0998535230755806,0.0,0.0625,0.6875
62,model.layers.8.mlp.down_proj,tx.8.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0654296875,0.10009765625,0.13818359375,0.0999581515789032,0.0,0.1015625,0.328125,0.0999581515789032,0.0,0.0625,0.625
63,model.layers.9.self_attn.q_proj,tx.9.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0712890625,0.099609375,0.1318359375,0.099853515625,0.0,0.1015625,0.2578125,0.099853515625,0.0,0.0625,0.5625
64,model.layers.9.self_attn.k_proj,tx.9.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0712890625,0.099609375,0.1298828125,0.099853515625,0.0,0.1015625,0.359375,0.099853515625,0.0,0.0625,0.5625
65,model.layers.9.self_attn.v_proj,tx.9.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07666015625,0.09912109375,0.16552734375,0.099853515625,0.0,0.1015625,0.9765625,0.099853515625,0.0,0.0625,0.5625
66,model.layers.9.self_attn.o_proj,tx.9.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0458984375,0.09619140625,0.21240234375,0.099853515625,0.0,0.09375,0.328125,0.099853515625,0.0,0.0625,0.625
67,model.layers.9.mlp.gate_proj,tx.9.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.06884765625,0.099609375,0.130859375,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.625
68,model.layers.9.mlp.up_proj,tx.9.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07470703125,0.099609375,0.12841796875,0.0998535230755806,0.0,0.1015625,0.2578125,0.0998535230755806,0.0,0.0625,0.625
69,model.layers.9.mlp.down_proj,tx.9.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06787109375,0.099609375,0.130859375,0.0999581515789032,0.0,0.1015625,0.3046875,0.0999581515789032,0.0,0.0625,0.625
70,model.layers.10.self_attn.q_proj,tx.10.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0732421875,0.099609375,0.13427734375,0.099853515625,0.0,0.1015625,0.25,0.099853515625,0.0,0.0625,0.5625
71,model.layers.10.self_attn.k_proj,tx.10.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0703125,0.099609375,0.126953125,0.099853515625,0.0,0.1015625,0.3359375,0.099853515625,0.0,0.0625,0.5
72,model.layers.10.self_attn.v_proj,tx.10.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07568359375,0.09912109375,0.1611328125,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5625
73,model.layers.10.self_attn.o_proj,tx.10.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.04248046875,0.0888671875,0.23583984375,0.099853515625,0.0,0.09375,0.3515625,0.099853515625,0.0,0.0625,0.6875
74,model.layers.10.mlp.gate_proj,tx.10.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.099609375,0.1318359375,0.0998535230755806,0.0,0.1015625,0.2421875,0.0998535230755806,0.0,0.0625,0.5625
75,model.layers.10.mlp.up_proj,tx.10.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07080078125,0.099609375,0.1279296875,0.0998535230755806,0.0,0.1015625,0.2421875,0.0998535230755806,0.0,0.0625,0.6875
76,model.layers.10.mlp.down_proj,tx.10.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06689453125,0.10009765625,0.13525390625,0.0999581515789032,0.0,0.1015625,0.3125,0.0999581515789032,0.0,0.0625,0.6875
77,model.layers.11.self_attn.q_proj,tx.11.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0693359375,0.099609375,0.13134765625,0.099853515625,0.0,0.1015625,0.2890625,0.099853515625,0.0,0.0625,0.625
78,model.layers.11.self_attn.k_proj,tx.11.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.072265625,0.099609375,0.1318359375,0.099853515625,0.0,0.1015625,0.4140625,0.099853515625,0.0,0.0625,0.625
79,model.layers.11.self_attn.v_proj,tx.11.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.06982421875,0.099609375,0.14892578125,0.099853515625,0.0,0.1015625,0.9765625,0.099853515625,0.0,0.0625,0.5625
80,model.layers.11.self_attn.o_proj,tx.11.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.04931640625,0.0927734375,0.23876953125,0.099853515625,0.0,0.09375,0.359375,0.099853515625,0.0,0.0625,0.625
81,model.layers.11.mlp.gate_proj,tx.11.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07177734375,0.099609375,0.12939453125,0.0998535230755806,0.0,0.1015625,0.2421875,0.0998535230755806,0.0,0.0625,0.625
82,model.layers.11.mlp.up_proj,tx.11.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.10009765625,0.126953125,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.625
83,model.layers.11.mlp.down_proj,tx.11.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06591796875,0.099609375,0.1337890625,0.0999581515789032,0.0,0.1015625,0.3515625,0.0999581515789032,0.0,0.0625,0.625
84,model.layers.12.self_attn.q_proj,tx.12.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.072265625,0.099609375,0.12890625,0.099853515625,0.0,0.1015625,0.2734375,0.099853515625,0.0,0.0625,0.5625
85,model.layers.12.self_attn.k_proj,tx.12.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07470703125,0.10009765625,0.13232421875,0.099853515625,0.0,0.1015625,0.375,0.099853515625,0.0,0.0625,0.5
86,model.layers.12.self_attn.v_proj,tx.12.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.076171875,0.099609375,0.1611328125,0.099853515625,0.0,0.1015625,0.9765625,0.099853515625,0.0,0.0625,0.5625
87,model.layers.12.self_attn.o_proj,tx.12.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03515625,0.099609375,0.16357421875,0.099853515625,0.0,0.1015625,0.3046875,0.099853515625,0.0,0.0625,0.6875
88,model.layers.12.mlp.gate_proj,tx.12.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.06982421875,0.099609375,0.1328125,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.5625
89,model.layers.12.mlp.up_proj,tx.12.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07177734375,0.099609375,0.12646484375,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.625
90,model.layers.12.mlp.down_proj,tx.12.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06982421875,0.10009765625,0.1396484375,0.0999581515789032,0.0,0.1015625,0.296875,0.0999581515789032,0.0,0.0625,0.5625
91,model.layers.13.self_attn.q_proj,tx.13.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06884765625,0.099609375,0.12841796875,0.099853515625,0.0,0.1015625,0.265625,0.099853515625,0.0,0.0625,0.5625
92,model.layers.13.self_attn.k_proj,tx.13.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0693359375,0.099609375,0.12744140625,0.099853515625,0.0,0.1015625,0.453125,0.099853515625,0.0,0.0625,0.5625
93,model.layers.13.self_attn.v_proj,tx.13.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07568359375,0.09912109375,0.171875,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5625
94,model.layers.13.self_attn.o_proj,tx.13.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.04296875,0.095703125,0.20458984375,0.099853515625,0.0,0.09375,0.3359375,0.099853515625,0.0,0.0625,0.625
95,model.layers.13.mlp.gate_proj,tx.13.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07080078125,0.099609375,0.13671875,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.625
96,model.layers.13.mlp.up_proj,tx.13.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07080078125,0.099609375,0.1279296875,0.0998535230755806,0.0,0.1015625,0.234375,0.0998535230755806,0.0,0.0625,0.625
97,model.layers.13.mlp.down_proj,tx.13.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06884765625,0.099609375,0.13427734375,0.0999581515789032,0.0,0.1015625,0.34375,0.0999581515789032,0.0,0.0625,0.625
98,model.layers.14.self_attn.q_proj,tx.14.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06787109375,0.099609375,0.12646484375,0.099853515625,0.0,0.1015625,0.25,0.099853515625,0.0,0.0625,0.5625
99,model.layers.14.self_attn.k_proj,tx.14.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.06884765625,0.099609375,0.1328125,0.099853515625,0.0,0.1015625,0.421875,0.099853515625,0.0,0.0625,0.5625
100,model.layers.14.self_attn.v_proj,tx.14.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07666015625,0.099609375,0.16015625,0.099853515625,0.0,0.1015625,0.9765625,0.099853515625,0.0,0.0625,0.5
101,model.layers.14.self_attn.o_proj,tx.14.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.044921875,0.09521484375,0.2099609375,0.099853515625,0.0,0.09375,0.3515625,0.099853515625,0.0,0.0625,0.6875
102,model.layers.14.mlp.gate_proj,tx.14.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.068359375,0.099609375,0.130859375,0.0998535230755806,0.0,0.1015625,0.2421875,0.0998535230755806,0.0,0.0625,0.625
103,model.layers.14.mlp.up_proj,tx.14.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0732421875,0.099609375,0.12890625,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.625
104,model.layers.14.mlp.down_proj,tx.14.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0673828125,0.099609375,0.13525390625,0.0999581515789032,0.0,0.1015625,0.2890625,0.0999581515789032,0.0,0.0625,0.5625
105,model.layers.15.self_attn.q_proj,tx.15.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.07275390625,0.099609375,0.12939453125,0.099853515625,0.0,0.1015625,0.296875,0.099853515625,0.0,0.0625,0.6875
106,model.layers.15.self_attn.k_proj,tx.15.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0615234375,0.099609375,0.1279296875,0.099853515625,0.0,0.1015625,0.3359375,0.099853515625,0.0,0.0625,0.5625
107,model.layers.15.self_attn.v_proj,tx.15.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07080078125,0.09912109375,0.16259765625,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5625
108,model.layers.15.self_attn.o_proj,tx.15.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03662109375,0.0869140625,0.2470703125,0.099853515625,0.0,0.0859375,0.390625,0.099853515625,0.0,0.0625,0.75
109,model.layers.15.mlp.gate_proj,tx.15.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07080078125,0.099609375,0.13037109375,0.0998535230755806,0.0,0.1015625,0.2578125,0.0998535230755806,0.0,0.0625,0.625
110,model.layers.15.mlp.up_proj,tx.15.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0654296875,0.099609375,0.12841796875,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.625
111,model.layers.15.mlp.down_proj,tx.15.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06494140625,0.10009765625,0.13623046875,0.0999581515789032,0.0,0.1015625,0.3046875,0.0999581515789032,0.0,0.0625,0.5625
112,model.layers.16.self_attn.q_proj,tx.16.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0732421875,0.10009765625,0.13232421875,0.099853515625,0.0,0.1015625,0.25,0.099853515625,0.0,0.0625,0.625
113,model.layers.16.self_attn.k_proj,tx.16.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0703125,0.10009765625,0.12548828125,0.099853515625,0.0,0.1015625,0.3828125,0.099853515625,0.0,0.0625,0.5625
114,model.layers.16.self_attn.v_proj,tx.16.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0712890625,0.099609375,0.16748046875,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5
115,model.layers.16.self_attn.o_proj,tx.16.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03173828125,0.09375,0.21484375,0.099853515625,0.0,0.09375,0.3359375,0.099853515625,0.0,0.0625,0.6875
116,model.layers.16.mlp.gate_proj,tx.16.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07373046875,0.099609375,0.12939453125,0.0998535230755806,0.0,0.1015625,0.2421875,0.0998535230755806,0.0,0.0625,0.625
117,model.layers.16.mlp.up_proj,tx.16.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07275390625,0.099609375,0.12939453125,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.5625
118,model.layers.16.mlp.down_proj,tx.16.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06982421875,0.10009765625,0.1376953125,0.0999581515789032,0.0,0.1015625,0.3046875,0.0999581515789032,0.0,0.0625,0.625
119,model.layers.17.self_attn.q_proj,tx.17.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06787109375,0.099609375,0.1279296875,0.099853515625,0.0,0.1015625,0.2421875,0.099853515625,0.0,0.0625,0.5625
120,model.layers.17.self_attn.k_proj,tx.17.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.072265625,0.099609375,0.13720703125,0.099853515625,0.0,0.1015625,0.3828125,0.099853515625,0.0,0.0625,0.5625
121,model.layers.17.self_attn.v_proj,tx.17.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07421875,0.099609375,0.16943359375,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5
122,model.layers.17.self_attn.o_proj,tx.17.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03515625,0.09033203125,0.23583984375,0.099853515625,0.0,0.09375,0.359375,0.099853515625,0.0,0.0625,0.6875
123,model.layers.17.mlp.gate_proj,tx.17.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.099609375,0.12890625,0.0998535230755806,0.0,0.1015625,0.2421875,0.0998535230755806,0.0,0.0625,0.625
124,model.layers.17.mlp.up_proj,tx.17.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07470703125,0.099609375,0.12841796875,0.0998535230755806,0.0,0.1015625,0.2734375,0.0998535230755806,0.0,0.0625,0.625
125,model.layers.17.mlp.down_proj,tx.17.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0703125,0.099609375,0.1337890625,0.0999581515789032,0.0,0.1015625,0.3671875,0.0999581515789032,0.0,0.0625,0.625
126,model.layers.18.self_attn.q_proj,tx.18.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.072265625,0.099609375,0.12890625,0.099853515625,0.0,0.1015625,0.2890625,0.099853515625,0.0,0.0625,0.625
127,model.layers.18.self_attn.k_proj,tx.18.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0712890625,0.10009765625,0.13720703125,0.099853515625,0.0,0.1015625,0.46875,0.099853515625,0.0,0.0625,0.5625
128,model.layers.18.self_attn.v_proj,tx.18.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07373046875,0.099609375,0.16748046875,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5
129,model.layers.18.self_attn.o_proj,tx.18.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0302734375,0.076171875,0.31396484375,0.099853515625,0.0,0.0859375,0.4765625,0.099853515625,0.0,0.0625,0.8125
130,model.layers.18.mlp.gate_proj,tx.18.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.099609375,0.130859375,0.0998535230755806,0.0,0.1015625,0.265625,0.0998535230755806,0.0,0.0625,0.625
131,model.layers.18.mlp.up_proj,tx.18.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.072265625,0.099609375,0.1298828125,0.0998535230755806,0.0,0.1015625,0.2734375,0.0998535230755806,0.0,0.0625,0.625
132,model.layers.18.mlp.down_proj,tx.18.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0693359375,0.10009765625,0.1337890625,0.0999581515789032,0.0,0.1015625,0.28125,0.0999581515789032,0.0,0.0625,0.625
133,model.layers.19.self_attn.q_proj,tx.19.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0732421875,0.099609375,0.12744140625,0.099853515625,0.0,0.1015625,0.2578125,0.099853515625,0.0,0.0625,0.5625
134,model.layers.19.self_attn.k_proj,tx.19.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07080078125,0.10009765625,0.1357421875,0.099853515625,0.0,0.1015625,0.3828125,0.099853515625,0.0,0.0625,0.5625
135,model.layers.19.self_attn.v_proj,tx.19.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0732421875,0.099609375,0.16845703125,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
136,model.layers.19.self_attn.o_proj,tx.19.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.02490234375,0.09521484375,0.2490234375,0.099853515625,0.0,0.09375,0.3984375,0.099853515625,0.0,0.0625,0.8125
137,model.layers.19.mlp.gate_proj,tx.19.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0732421875,0.099609375,0.126953125,0.0998535230755806,0.0,0.1015625,0.25,0.0998535230755806,0.0,0.0625,0.625
138,model.layers.19.mlp.up_proj,tx.19.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07421875,0.099609375,0.12939453125,0.0998535230755806,0.0,0.1015625,0.296875,0.0998535230755806,0.0,0.0625,0.625
139,model.layers.19.mlp.down_proj,tx.19.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06396484375,0.10009765625,0.1357421875,0.0999581515789032,0.0,0.1015625,0.2734375,0.0999581515789032,0.0,0.0625,0.625
140,model.layers.20.self_attn.q_proj,tx.20.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0712890625,0.10009765625,0.13134765625,0.099853515625,0.0,0.1015625,0.2890625,0.099853515625,0.0,0.0625,0.5625
141,model.layers.20.self_attn.k_proj,tx.20.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.072265625,0.10009765625,0.1328125,0.099853515625,0.0,0.1015625,0.4296875,0.099853515625,0.0,0.0625,0.625
142,model.layers.20.self_attn.v_proj,tx.20.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07373046875,0.099609375,0.17626953125,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5
143,model.layers.20.self_attn.o_proj,tx.20.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03662109375,0.0859375,0.2783203125,0.099853515625,0.0,0.0859375,0.46875,0.099853515625,0.0,0.0625,0.75
144,model.layers.20.mlp.gate_proj,tx.20.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.099609375,0.1279296875,0.0998535230755806,0.0,0.1015625,0.265625,0.0998535230755806,0.0,0.0625,0.625
145,model.layers.20.mlp.up_proj,tx.20.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07275390625,0.099609375,0.1259765625,0.0998535230755806,0.0,0.1015625,0.3125,0.0998535230755806,0.0,0.0625,0.5625
146,model.layers.20.mlp.down_proj,tx.20.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06884765625,0.10009765625,0.1318359375,0.0999581515789032,0.0,0.1015625,0.2734375,0.0999581515789032,0.0,0.0625,0.625
147,model.layers.21.self_attn.q_proj,tx.21.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06884765625,0.10009765625,0.1298828125,0.099853515625,0.0,0.1015625,0.28125,0.099853515625,0.0,0.0625,0.5625
148,model.layers.21.self_attn.k_proj,tx.21.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07373046875,0.099609375,0.12744140625,0.099853515625,0.0,0.1015625,0.375,0.099853515625,0.0,0.0625,0.5
149,model.layers.21.self_attn.v_proj,tx.21.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07275390625,0.09912109375,0.16162109375,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.625
150,model.layers.21.self_attn.o_proj,tx.21.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.02734375,0.08349609375,0.3203125,0.099853515625,0.0,0.09375,0.5234375,0.099853515625,0.0,0.0625,0.75
151,model.layers.21.mlp.gate_proj,tx.21.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07275390625,0.099609375,0.13134765625,0.0998535230755806,0.0,0.1015625,0.265625,0.0998535230755806,0.0,0.0625,0.5625
152,model.layers.21.mlp.up_proj,tx.21.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.06884765625,0.099609375,0.12646484375,0.0998535230755806,0.0,0.1015625,0.2734375,0.0998535230755806,0.0,0.0625,0.625
153,model.layers.21.mlp.down_proj,tx.21.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06689453125,0.10009765625,0.13623046875,0.0999581515789032,0.0,0.1015625,0.296875,0.0999581515789032,0.0,0.0625,0.5625
154,model.layers.22.self_attn.q_proj,tx.22.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0703125,0.10009765625,0.12646484375,0.099853515625,0.0,0.1015625,0.2421875,0.099853515625,0.0,0.0625,0.5625
155,model.layers.22.self_attn.k_proj,tx.22.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0732421875,0.099609375,0.13134765625,0.099853515625,0.0,0.1015625,0.4609375,0.099853515625,0.0,0.0625,0.5625
156,model.layers.22.self_attn.v_proj,tx.22.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0732421875,0.099609375,0.1669921875,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
157,model.layers.22.self_attn.o_proj,tx.22.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0341796875,0.095703125,0.22021484375,0.099853515625,0.0,0.09375,0.3515625,0.099853515625,0.0,0.0625,0.625
158,model.layers.22.mlp.gate_proj,tx.22.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0732421875,0.099609375,0.13134765625,0.0998535230755806,0.0,0.1015625,0.265625,0.0998535230755806,0.0,0.0625,0.625
159,model.layers.22.mlp.up_proj,tx.22.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07568359375,0.099609375,0.12744140625,0.0998535230755806,0.0,0.1015625,0.28125,0.0998535230755806,0.0,0.0625,0.6875
160,model.layers.22.mlp.down_proj,tx.22.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06494140625,0.10009765625,0.13525390625,0.0999581515789032,0.0,0.1015625,0.265625,0.0999581515789032,0.0,0.0625,0.625
161,model.layers.23.self_attn.q_proj,tx.23.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.07080078125,0.10009765625,0.13330078125,0.099853515625,0.0,0.1015625,0.2421875,0.099853515625,0.0,0.0625,0.625
162,model.layers.23.self_attn.k_proj,tx.23.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0732421875,0.099609375,0.1298828125,0.099853515625,0.0,0.1015625,0.40625,0.099853515625,0.0,0.0625,0.5625
163,model.layers.23.self_attn.v_proj,tx.23.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07470703125,0.09912109375,0.16552734375,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5625
164,model.layers.23.self_attn.o_proj,tx.23.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03173828125,0.0947265625,0.22021484375,0.099853515625,0.0,0.09375,0.328125,0.099853515625,0.0,0.0625,0.625
165,model.layers.23.mlp.gate_proj,tx.23.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07470703125,0.099609375,0.13037109375,0.0998535230755806,0.0,0.1015625,0.296875,0.0998535230755806,0.0,0.0625,0.5625
166,model.layers.23.mlp.up_proj,tx.23.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0703125,0.099609375,0.12890625,0.0998535230755806,0.0,0.1015625,0.3125,0.0998535230755806,0.0,0.0625,0.625
167,model.layers.23.mlp.down_proj,tx.23.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0673828125,0.10009765625,0.13671875,0.0999581515789032,0.0,0.1015625,0.265625,0.0999581515789032,0.0,0.0625,0.6875
168,model.layers.24.self_attn.q_proj,tx.24.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06982421875,0.099609375,0.126953125,0.099853515625,0.0,0.1015625,0.2421875,0.099853515625,0.0,0.0625,0.6875
169,model.layers.24.self_attn.k_proj,tx.24.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07470703125,0.099609375,0.13671875,0.099853515625,0.0,0.1015625,0.453125,0.099853515625,0.0,0.0625,0.5
170,model.layers.24.self_attn.v_proj,tx.24.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0673828125,0.09912109375,0.1669921875,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.625
171,model.layers.24.self_attn.o_proj,tx.24.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.03125,0.087890625,0.2255859375,0.099853515625,0.0,0.09375,0.3671875,0.099853515625,0.0,0.0625,0.6875
172,model.layers.24.mlp.gate_proj,tx.24.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07275390625,0.099609375,0.126953125,0.0998535230755806,0.0,0.1015625,0.2734375,0.0998535230755806,0.0,0.0625,0.625
173,model.layers.24.mlp.up_proj,tx.24.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0693359375,0.099609375,0.12841796875,0.0998535230755806,0.0,0.1015625,0.296875,0.0998535230755806,0.0,0.0625,0.5625
174,model.layers.24.mlp.down_proj,tx.24.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0654296875,0.10009765625,0.1357421875,0.0999581515789032,0.0,0.1015625,0.265625,0.0999581515789032,0.0,0.0625,0.625
175,model.layers.25.self_attn.q_proj,tx.25.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0732421875,0.099609375,0.1298828125,0.099853515625,0.0,0.1015625,0.2578125,0.099853515625,0.0,0.0625,0.625
176,model.layers.25.self_attn.k_proj,tx.25.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0712890625,0.099609375,0.13330078125,0.099853515625,0.0,0.1015625,0.3515625,0.099853515625,0.0,0.0625,0.5
177,model.layers.25.self_attn.v_proj,tx.25.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07177734375,0.099609375,0.16552734375,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
178,model.layers.25.self_attn.o_proj,tx.25.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.02685546875,0.080078125,0.3447265625,0.099853515625,0.0,0.0859375,0.5078125,0.099853515625,0.0,0.0625,0.8125
179,model.layers.25.mlp.gate_proj,tx.25.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0712890625,0.099609375,0.12939453125,0.0998535230755806,0.0,0.1015625,0.296875,0.0998535230755806,0.0,0.0625,0.625
180,model.layers.25.mlp.up_proj,tx.25.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.072265625,0.099609375,0.13232421875,0.0998535230755806,0.0,0.1015625,0.3125,0.0998535230755806,0.0,0.0625,0.5625
181,model.layers.25.mlp.down_proj,tx.25.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0654296875,0.10009765625,0.140625,0.0999581515789032,0.0,0.1015625,0.265625,0.0999581515789032,0.0,0.0625,0.625
182,model.layers.26.self_attn.q_proj,tx.26.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0732421875,0.10009765625,0.1279296875,0.099853515625,0.0,0.1015625,0.2578125,0.099853515625,0.0,0.0625,0.5625
183,model.layers.26.self_attn.k_proj,tx.26.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07568359375,0.10009765625,0.130859375,0.099853515625,0.0,0.1015625,0.3828125,0.099853515625,0.0,0.0625,0.5
184,model.layers.26.self_attn.v_proj,tx.26.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07373046875,0.099609375,0.1689453125,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
185,model.layers.26.self_attn.o_proj,tx.26.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.02978515625,0.0849609375,0.23291015625,0.099853515625,0.0,0.0859375,0.375,0.099853515625,0.0,0.0625,0.6875
186,model.layers.26.mlp.gate_proj,tx.26.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.072265625,0.099609375,0.1298828125,0.0998535230755806,0.0,0.1015625,0.296875,0.0998535230755806,0.0,0.0625,0.625
187,model.layers.26.mlp.up_proj,tx.26.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0712890625,0.099609375,0.12841796875,0.0998535230755806,0.0,0.1015625,0.28125,0.0998535230755806,0.0,0.0625,0.5625
188,model.layers.26.mlp.down_proj,tx.26.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06396484375,0.10009765625,0.13525390625,0.0999581515789032,0.0,0.1015625,0.265625,0.0999581515789032,0.0,0.0625,0.625
189,model.layers.27.self_attn.q_proj,tx.27.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.068359375,0.099609375,0.17431640625,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
190,model.layers.27.self_attn.k_proj,tx.27.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07080078125,0.099609375,0.1611328125,0.099853515625,0.0,0.1015625,0.96875,0.099853515625,0.0,0.0625,0.5
191,model.layers.27.self_attn.v_proj,tx.27.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0703125,0.099609375,0.16552734375,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
192,model.layers.27.self_attn.o_proj,tx.27.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0302734375,0.087890625,0.2216796875,0.099853515625,0.0,0.0859375,0.3671875,0.099853515625,0.0,0.0625,0.6875
193,model.layers.27.mlp.gate_proj,tx.27.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07470703125,0.099609375,0.12939453125,0.0998535230755806,0.0,0.1015625,0.28125,0.0998535230755806,0.0,0.0625,0.5625
194,model.layers.27.mlp.up_proj,tx.27.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0712890625,0.099609375,0.12744140625,0.0998535230755806,0.0,0.1015625,0.28125,0.0998535230755806,0.0,0.0625,0.5625
195,model.layers.27.mlp.down_proj,tx.27.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.068359375,0.10009765625,0.1337890625,0.0999581515789032,0.0,0.1015625,0.265625,0.0999581515789032,0.0,0.0625,0.625
196,model.layers.28.self_attn.q_proj,tx.28.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0703125,0.10009765625,0.1279296875,0.099853515625,0.0,0.1015625,0.25,0.099853515625,0.0,0.0625,0.5625
197,model.layers.28.self_attn.k_proj,tx.28.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07177734375,0.10009765625,0.12890625,0.099853515625,0.0,0.1015625,0.3515625,0.099853515625,0.0,0.0625,0.5625
198,model.layers.28.self_attn.v_proj,tx.28.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0703125,0.099609375,0.17333984375,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5625
199,model.layers.28.self_attn.o_proj,tx.28.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.025390625,0.095703125,0.25634765625,0.099853515625,0.0,0.09375,0.3671875,0.099853515625,0.0,0.0625,0.75
200,model.layers.28.mlp.gate_proj,tx.28.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07275390625,0.10009765625,0.130859375,0.0998535230755806,0.0,0.1015625,0.296875,0.0998535230755806,0.0,0.0625,0.625
201,model.layers.28.mlp.up_proj,tx.28.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0732421875,0.099609375,0.1318359375,0.0998535230755806,0.0,0.1015625,0.296875,0.0998535230755806,0.0,0.0625,0.625
202,model.layers.28.mlp.down_proj,tx.28.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06396484375,0.10009765625,0.1396484375,0.0999581515789032,0.0,0.1015625,0.265625,0.0999581515789032,0.0,0.0625,0.6875
203,model.layers.29.self_attn.q_proj,tx.29.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0693359375,0.099609375,0.18115234375,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.625
204,model.layers.29.self_attn.k_proj,tx.29.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0693359375,0.099609375,0.16259765625,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
205,model.layers.29.self_attn.v_proj,tx.29.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07666015625,0.09912109375,0.169921875,0.099853515625,0.0,0.1015625,1.0,0.099853515625,0.0,0.0625,0.5625
206,model.layers.29.self_attn.o_proj,tx.29.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.01806640625,0.1064453125,0.20068359375,0.099853515625,0.0,0.09375,0.328125,0.099853515625,0.0,0.0625,0.625
207,model.layers.29.mlp.gate_proj,tx.29.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0712890625,0.099609375,0.126953125,0.0998535230755806,0.0,0.1015625,0.2734375,0.0998535230755806,0.0,0.0625,0.625
208,model.layers.29.mlp.up_proj,tx.29.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.072265625,0.099609375,0.13232421875,0.0998535230755806,0.0,0.1015625,0.2890625,0.0998535230755806,0.0,0.0625,0.625
209,model.layers.29.mlp.down_proj,tx.29.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.06689453125,0.10009765625,0.134765625,0.0999581515789032,0.0,0.1015625,0.2578125,0.0999581515789032,0.0,0.0625,0.5625
210,model.layers.30.self_attn.q_proj,tx.30.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.07080078125,0.10009765625,0.13427734375,0.099853515625,0.0,0.1015625,0.2578125,0.099853515625,0.0,0.0625,0.625
211,model.layers.30.self_attn.k_proj,tx.30.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.06884765625,0.10009765625,0.12646484375,0.099853515625,0.0,0.1015625,0.34375,0.099853515625,0.0,0.0625,0.5625
212,model.layers.30.self_attn.v_proj,tx.30.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.07421875,0.09912109375,0.18212890625,0.099853515625,0.0,0.1015625,0.9921875,0.099853515625,0.0,0.0625,0.5625
213,model.layers.30.self_attn.o_proj,tx.30.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.0244140625,0.08984375,0.337890625,0.099853515625,0.0,0.0859375,0.4609375,0.099853515625,0.0,0.0625,0.75
214,model.layers.30.mlp.gate_proj,tx.30.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.07177734375,0.099609375,0.13134765625,0.0998535230755806,0.0,0.1015625,0.328125,0.0998535230755806,0.0,0.0625,0.625
215,model.layers.30.mlp.up_proj,tx.30.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0732421875,0.099609375,0.12841796875,0.0998535230755806,0.0,0.1015625,0.3125,0.0998535230755806,0.0,0.0625,0.625
216,model.layers.30.mlp.down_proj,tx.30.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.0625,0.10009765625,0.14111328125,0.0999581515789032,0.0,0.1015625,0.28125,0.0999581515789032,0.0,0.0625,0.625
217,model.layers.31.self_attn.q_proj,tx.31.attn.q,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.06982421875,0.099609375,0.1328125,0.099853515625,0.0,0.1015625,0.375,0.099853515625,0.0,0.0625,0.625
218,model.layers.31.self_attn.k_proj,tx.31.attn.k,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.06689453125,0.10009765625,0.12646484375,0.099853515625,0.0,0.1015625,0.3828125,0.099853515625,0.0,0.0625,0.5625
219,model.layers.31.self_attn.v_proj,tx.31.attn.v,Linear,weight,"[1024, 4096]",4194304,3775488,0.099853515625,"(128, 16)",8 x 256,2048,0.099853515625,0.0673828125,0.099609375,0.15771484375,0.099853515625,0.0,0.1015625,0.90625,0.099853515625,0.0,0.0625,0.5625
220,model.layers.31.self_attn.o_proj,tx.31.attn.o,Linear,weight,"[4096, 4096]",16777216,15101952,0.099853515625,"(128, 16)",32 x 256,8192,0.099853515625,0.01025390625,0.08642578125,0.22802734375,0.099853515625,0.0,0.0859375,0.3671875,0.099853515625,0.0,0.0625,0.6875
221,model.layers.31.mlp.gate_proj,tx.31.mlp.gate,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.0712890625,0.099609375,0.13037109375,0.0998535230755806,0.0,0.1015625,0.328125,0.0998535230755806,0.0,0.0625,0.6875
222,model.layers.31.mlp.up_proj,tx.31.mlp.up,Linear,weight,"[14336, 4096]",58720256,52856832,0.09985345602035522,"(128, 16)",112 x 256,28672,0.0998535230755806,0.06982421875,0.099609375,0.12890625,0.0998535230755806,0.0,0.1015625,0.3125,0.0998535230755806,0.0,0.0625,0.625
223,model.layers.31.mlp.down_proj,tx.31.mlp.down,Linear,weight,"[4096, 14336]",58720256,52850688,0.09995812177658081,"(128, 16)",32 x 896,28672,0.0999581515789032,0.05859375,0.10009765625,0.15234375,0.0999581515789032,0.0,0.1015625,0.3359375,0.0999581515789032,0.0,0.0625,0.5625
224,lm_head,lm_head,Linear,weight,"[128256, 4096]",525336576,525336576,0.0,"(128, 16)",1002 x 256,256512,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0