diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json" new file mode 100644--- /dev/null +++ "b/reduced_ppl_average.json" @@ -0,0 +1,33680 @@ +{ + "original_ppl": 3.5889670848846436, + "reduced_ppl": [ + { + "bit": 1, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 21188.080078125 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 21507.11328125 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 20110.830078125 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 17031.22265625 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 21325.625 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 22083.509765625 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 19715.720703125 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 21485.3828125 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 21357.619140625 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 21229.1796875 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 18235.662109375 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 20092.197265625 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 20723.626953125 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 21363.078125 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 21859.53515625 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 21388.255859375 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 23075.7890625 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 22187.15625 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 21149.05859375 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 20982.552734375 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 21100.947265625 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 21260.927734375 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 21491.61328125 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 19500.5625 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 22569.806640625 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 21019.884765625 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 20949.720703125 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 20938.734375 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 21176.140625 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 21171.80078125 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 19567.9453125 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 21054.373046875 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 20710.15234375 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 21046.443359375 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 21028.546875 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 21429.2734375 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 21416.13671875 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 21224.421875 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 20703.75390625 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 21001.130859375 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 20810.689453125 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 21414.890625 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 21056.783203125 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 21288.27734375 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 21985.185546875 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 21147.525390625 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 20685.240234375 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 22052.802734375 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 20772.896484375 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 21137.54296875 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 21494.01171875 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 22372.45703125 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 21929.96484375 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 21850.84375 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 20725.880859375 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 21633.998046875 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 21364.7109375 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 21331.21875 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 22104.2421875 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 21250.9140625 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 21253.060546875 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 21452.17578125 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 21225.435546875 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 22018.0234375 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 21259.041015625 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 21475.775390625 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 22780.8203125 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 21065.51953125 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 20531.587890625 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 21159.0625 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 20956.31640625 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 21341.6171875 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 21552.61328125 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 23110.830078125 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 20643.265625 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 21109.0390625 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 20632.48046875 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 21203.28125 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 21220.818359375 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 21761.21484375 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 21016.517578125 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 20669.23046875 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 21539.75 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 22339.943359375 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 21634.84375 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 21276.890625 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 23412.193359375 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 23611.7109375 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 22845.12890625 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 21595.986328125 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 21236.99609375 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 21862.560546875 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 21377.38671875 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 24828.98828125 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 20674.88671875 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 20898.23828125 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 20445.966796875 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 20182.359375 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 20496.765625 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 20505.994140625 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 22029.99609375 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 16201.755859375 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 22489.921875 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 21489.11328125 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 21110.75 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 20777.73046875 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 21277.560546875 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 24620.99609375 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 19499.150390625 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 21273.724609375 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 21399.62109375 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 20567.060546875 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 21429.4375 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 21209.388671875 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 23761.4296875 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 22053.265625 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 20012.60546875 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 20855.51171875 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 17321.794921875 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 21573.341796875 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 21940.96875 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 28286.912109375 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 18625.5703125 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 19470.125 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 21710.904296875 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 20286.5078125 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 21511.134765625 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 20891.302734375 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 19786.072265625 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 25939.958984375 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 20720.20703125 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 20727.85546875 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 22093.62109375 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 21050.478515625 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 20432.478515625 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 23068.177734375 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 17286.083984375 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 21471.33203125 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 20273.955078125 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 19921.75390625 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 20936.798828125 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 20978.55078125 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 20239.0078125 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 16796.146484375 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 21711.587890625 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 21914.09765625 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 21841.865234375 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 21723.310546875 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 21555.9453125 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 25035.494140625 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 23986.306640625 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 21940.3828125 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 20579.65625 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 20480.861328125 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 20903.439453125 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 20932.185546875 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 22755.958984375 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 22541.884765625 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 21653.318359375 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 20392.8046875 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 20796.958984375 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 21248.765625 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 21007.66015625 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 21142.341796875 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 19766.17578125 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 21469.16015625 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 20484.78515625 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 20816.763671875 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 20933.32421875 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 20995.001953125 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 20636.1015625 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 22674.65625 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 21464.822265625 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 20596.306640625 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 22281.388671875 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 21209.2890625 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 21260.318359375 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 21882.771484375 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 20282.79296875 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 22490.220703125 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 20207.205078125 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 20807.552734375 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 21265.8359375 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 21007.30078125 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 20513.11328125 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 21582.849609375 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 23047.662109375 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 20811.7421875 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 21672.986328125 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 21246.576171875 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 21311.109375 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 21356.439453125 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 21776.412109375 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 20390.916015625 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 21756.150390625 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 20295.390625 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 21555.533203125 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 21623.849609375 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 20264.2890625 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 21611.35546875 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 21575.072265625 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 20574.947265625 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 21042.30859375 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 21491.388671875 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 21223.328125 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 21142.40234375 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 21555.69921875 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 21743.20703125 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 20545.33984375 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 20309.15625 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 20886.94140625 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 21029.34765625 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 22449.720703125 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 22464.326171875 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 21834.220703125 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 21394.171875 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 21281.05078125 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 21487.71875 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 20974.7109375 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 21599.529296875 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 24651.962890625 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 21947.560546875 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 22286.658203125 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 20780.822265625 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 21540.962890625 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 21302.83984375 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 21780.5234375 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 20080.337890625 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 21722.73046875 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 23338.138671875 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 21399.333984375 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 21857.658203125 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 21081.41796875 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 21487.80078125 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 21448.533203125 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 20473.673828125 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 22222.18359375 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 20200.421875 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 21422.3671875 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 21577.7265625 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 20939.21484375 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 19311.0546875 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 20061.810546875 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 21112.1796875 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 21474.546875 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 21082.884765625 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 20450.958984375 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 21350.900390625 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 22016.17578125 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 21559.76953125 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 20349.423828125 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 21367.236328125 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 21507.03125 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 21835.513671875 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 22577.857421875 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 20836.345703125 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 21133.310546875 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 21191.455078125 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 20561.375 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 21434.955078125 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 21494.236328125 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 21486.12109375 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 21522.154296875 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 21591.869140625 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 20596.1875 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 21724.3046875 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 21537.73828125 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 21060.056640625 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 20378.921875 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 21125.853515625 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 21154.583984375 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 20748.859375 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 21640.3125 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 21301.072265625 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 21446.794921875 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 20997.166015625 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 22515.54296875 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 20618.001953125 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 20777.73046875 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 22076.939453125 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 21357.51953125 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 21108.111328125 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 22000.685546875 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 20541.7734375 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 21658.109375 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 20189.001953125 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 21589.74609375 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 21353.34375 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 21456.4296875 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 21764.119140625 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 21489.154296875 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 20979.271484375 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 21162.67578125 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 21423.369140625 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 21334.23046875 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 21504.05859375 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 21616.42578125 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 21124.001953125 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 20071.55078125 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 20562.39453125 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 20841.13671875 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 21471.90625 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 21190.970703125 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 21952.982421875 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 21077.09375 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 21340.923828125 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 21728.447265625 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 21140.951171875 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 21464.37109375 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 20817.994140625 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 20304.953125 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 21218.37109375 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 21755.21875 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 21058.609375 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 21211.59375 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 21303.224609375 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 21420.69140625 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 20364.060546875 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 21023.072265625 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 21143.734375 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 21883.9609375 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 21835.26171875 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 21308.4453125 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 21292.705078125 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 21370.08984375 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 21380.119140625 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 20945.24609375 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 21349.39453125 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 20991.158203125 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 21228.755859375 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 21164.109375 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 21284.78515625 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 20602.455078125 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 21565.279296875 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 21632.86328125 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 21292.013671875 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 21070.421875 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 21338.767578125 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 20819.859375 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 20824.189453125 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 21068.29296875 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 21259.40625 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 21234.302734375 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 21221.24609375 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 21281.171875 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 21839.677734375 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 23035.5078125 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 21294.20703125 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 20622.0546875 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 21220.29296875 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 21306.35546875 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 21472.8671875 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 20760.103515625 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 21856.28515625 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 21028.0859375 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 21342.044921875 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 20698.3828125 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 21591.291015625 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 21412.11328125 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 21608.388671875 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 20974.388671875 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 20554.51171875 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 21624.7578125 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 21218.37109375 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 21488.96875 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 21068.875 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 21883.7109375 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 21779.52734375 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 20556.041015625 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 21769.765625 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 20799.658203125 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 20952.978515625 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 21001.451171875 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 21763.99609375 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 20934.7421875 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 20961.271484375 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 21031.052734375 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 21255.08984375 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 21320.095703125 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 21916.29296875 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 21941.513671875 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 21369.234375 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 21310.62109375 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 21170.205078125 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 21330.75 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 21359.453125 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 21245.98828125 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 21711.75390625 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 21100.8671875 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 21162.796875 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 21231.912109375 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 20784.70703125 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 21499.833984375 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 21081.0546875 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 20989.537109375 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 21823.16796875 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 21236.53125 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 21415.3203125 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 21544.517578125 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 21181.755859375 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 21024.3359375 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 21391.1328125 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 20971.849609375 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 21450.333984375 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 21031.353515625 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 21514.931640625 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 21225.677734375 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 20975.03125 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 21727.765625 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 20751.591796875 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 21142.865234375 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 21220.900390625 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 21144.177734375 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 21465.02734375 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 21491.490234375 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 21764.53515625 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 21021.2890625 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 21110.890625 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 21280.869140625 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 21306.109375 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 21318.142578125 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 21523.427734375 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 21735.97265625 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 23399.626953125 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 21726.025390625 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 20554.35546875 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 20796.443359375 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 21289.65625 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 21144.7421875 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 20745.45703125 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 21366.625 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 20820.734375 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 21242.990234375 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 21783.638671875 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 21209.44921875 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 21414.5234375 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 21088.95703125 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 20368.935546875 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 21502.109375 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 20755.134765625 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 21101.068359375 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 20878.017578125 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 21196.953125 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 20991.939453125 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 21255.130859375 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 21289.49609375 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 21829.599609375 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 20801.720703125 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 21231.529296875 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 20998.0859375 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 19772.01953125 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 22032.28515625 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 21454.017578125 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 21081.1953125 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 21135.287109375 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 21126.318359375 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 21540.203125 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 21123.658203125 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 21522.09375 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 20609.66796875 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 21069.677734375 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 21543.080078125 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 21372.271484375 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 21474.814453125 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 21973.259765625 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 20332.00390625 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 20785.619140625 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 21470.984375 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 21425.861328125 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 20974.150390625 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 20959.453125 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 21406.1328125 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 21418.03515625 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 21179.857421875 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 21530.22265625 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 21536.9375 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 21352.5703125 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 21883.064453125 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 22091.38671875 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 21688.947265625 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 20905.892578125 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 20844.693359375 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 21237.380859375 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 21100.041015625 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 21278.3125 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 21441.25390625 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 20805.052734375 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 20895.328125 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 20873.439453125 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 21420.26171875 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 22507.0625 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 20970.869140625 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 19957.142578125 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 20594.55859375 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 21115.884765625 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 21840.63671875 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 22266.328125 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 21533.259765625 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 21343.85546875 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 21529.03125 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 22012.3125 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 21135.38671875 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 19999.955078125 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 20936.9375 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 21389.05078125 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 21318.203125 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 21798.70703125 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 21630.65625 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 20363.146484375 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 20084.1875 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 20501.791015625 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 21277.3984375 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 21232.5 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 20900.58984375 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 21368.296875 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 21573.13671875 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 21142.765625 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 19692.345703125 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 21023.853515625 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 21036.126953125 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 21890.09765625 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 21159.708984375 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 20056.9140625 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 20642.87109375 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 21970.431640625 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 20949.98046875 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 22288.529296875 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 20812.13671875 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 23699.96484375 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 22202.439453125 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 22236.44921875 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 21622.365234375 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 22081.234375 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 21242.240234375 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 20781.416015625 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 21288.400390625 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 21200.9140625 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 23096.353515625 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 20319.75390625 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 21429.640625 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 20287.802734375 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 21331.7265625 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 20669.189453125 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 21269.689453125 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 20459.931640625 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 23201.548828125 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 22606.599609375 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 20575.849609375 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 20584.95703125 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 21127.970703125 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 19534.06640625 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 20416.9140625 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 20312.466796875 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 20217.556640625 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 19929.716796875 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 22085.572265625 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 21988.142578125 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 16646.185546875 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 20465.748046875 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 15951.1943359375 + } + ] + }, + { + "bit": 2, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 19.23235511779785 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 19.946475982666016 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 19.607023239135742 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 19.724580764770508 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 19.760160446166992 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 22.831995010375977 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 18.117769241333008 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 19.38628387451172 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 19.424415588378906 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 18.433713912963867 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 19.141603469848633 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 19.341575622558594 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 18.764795303344727 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 18.625791549682617 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 19.24675750732422 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 19.49931526184082 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 19.671314239501953 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 19.29741096496582 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 18.965078353881836 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 18.457401275634766 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 18.973833084106445 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 15.027917861938477 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 18.835464477539062 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 16.780004501342773 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 18.529701232910156 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 19.37194061279297 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 19.261810302734375 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 19.177326202392578 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 19.564071655273438 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 19.518863677978516 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 19.45000457763672 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 19.496259689331055 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 19.323137283325195 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 19.300172805786133 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 19.332874298095703 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 19.40731430053711 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 19.43332862854004 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 19.426036834716797 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 19.362342834472656 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 19.37186622619629 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 19.38475799560547 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 19.336843490600586 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 19.41978645324707 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 19.460933685302734 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 19.245546340942383 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 19.250930786132812 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 19.396726608276367 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 18.573503494262695 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 19.26388168334961 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 19.526159286499023 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 19.49813461303711 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 19.51691436767578 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 19.545059204101562 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 19.17987823486328 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 19.235849380493164 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 19.101125717163086 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 19.496122360229492 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 19.558839797973633 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 19.355976104736328 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 19.610631942749023 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 19.355995178222656 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 19.403400421142578 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 19.4201602935791 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 19.465299606323242 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 19.440956115722656 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 19.214031219482422 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 19.46047019958496 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 19.383787155151367 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 19.446287155151367 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 19.316505432128906 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 19.310583114624023 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 19.393564224243164 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 19.15888786315918 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 19.253461837768555 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 19.498849868774414 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 19.471961975097656 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 19.479873657226562 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 19.33439064025879 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 19.373807907104492 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 19.292747497558594 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 19.367876052856445 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 19.431840896606445 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 19.460817337036133 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 19.464691162109375 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 19.392915725708008 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 19.498491287231445 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 19.06902503967285 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 19.453231811523438 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 19.482316970825195 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 19.509479522705078 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 19.74064826965332 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 18.93439292907715 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 19.006093978881836 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 18.85584831237793 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 19.037612915039062 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 19.47408676147461 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 19.51532554626465 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 19.402130126953125 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 19.261369705200195 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 19.379837036132812 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 19.488704681396484 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 19.518264770507812 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 19.438711166381836 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 19.563997268676758 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 19.425151824951172 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 19.412090301513672 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 19.427528381347656 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 19.501583099365234 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 19.269062042236328 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 19.567575454711914 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 19.618240356445312 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 19.532392501831055 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 19.44793701171875 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 19.450700759887695 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 19.137161254882812 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 18.934301376342773 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 19.579736709594727 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 19.42156410217285 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 19.334552764892578 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 19.427846908569336 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 19.347505569458008 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 19.3753604888916 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 19.439443588256836 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 19.587646484375 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 19.547138214111328 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 19.555580139160156 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 19.322189331054688 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 19.423582077026367 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 19.291568756103516 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 19.188247680664062 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 19.36553192138672 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 19.335607528686523 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 19.27300453186035 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 19.38629150390625 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 19.319438934326172 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 19.551177978515625 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 19.48011589050293 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 19.359668731689453 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 19.470001220703125 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 19.290197372436523 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 19.559818267822266 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 19.49764060974121 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 19.488494873046875 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 19.446666717529297 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 19.539030075073242 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 19.50720977783203 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 19.408159255981445 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 19.362646102905273 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 19.44154930114746 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 19.186473846435547 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 19.197898864746094 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 19.35973358154297 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 19.382854461669922 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 19.43752098083496 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 19.362550735473633 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 19.450376510620117 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 19.46440887451172 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 19.307987213134766 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 19.2517147064209 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 19.792449951171875 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 19.593040466308594 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 19.30116081237793 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 19.368967056274414 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 19.1539306640625 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 19.138391494750977 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 19.22115135192871 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 19.74404525756836 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 19.446741104125977 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 19.677465438842773 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 19.59617042541504 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 19.153770446777344 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 18.964052200317383 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 19.147706985473633 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 19.73322296142578 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 19.508689880371094 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 19.47170066833496 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 19.39730453491211 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 19.497400283813477 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 19.143741607666016 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 18.97891616821289 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 19.363510131835938 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 19.158296585083008 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 19.17223358154297 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 19.274953842163086 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 19.36556053161621 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 19.453716278076172 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 19.181062698364258 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 19.54332160949707 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 19.44119644165039 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 19.4073429107666 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 19.428504943847656 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 19.43195343017578 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 19.107421875 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 19.253623962402344 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 19.41591453552246 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 19.43436622619629 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 19.800350189208984 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 19.509178161621094 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 19.636917114257812 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 19.36614990234375 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 19.329580307006836 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 19.334426879882812 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 19.44069480895996 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 19.409753799438477 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 19.3053035736084 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 19.44692039489746 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 19.124032974243164 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 19.193309783935547 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 19.462614059448242 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 19.464149475097656 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 19.563175201416016 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 19.51250457763672 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 19.401578903198242 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 19.223337173461914 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 19.32774543762207 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 19.283199310302734 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 19.374094009399414 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 19.393762588500977 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 19.452119827270508 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 19.468563079833984 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 19.45075798034668 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 19.427249908447266 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 19.41655921936035 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 19.453126907348633 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 19.493690490722656 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 19.432540893554688 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 19.485353469848633 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 19.368736267089844 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 19.397193908691406 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 19.427547454833984 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 19.41444206237793 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 19.371986389160156 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 19.523611068725586 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 19.50116539001465 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 19.193546295166016 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 19.41111946105957 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 19.409740447998047 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 19.386615753173828 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 19.419960021972656 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 19.41337776184082 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 19.364547729492188 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 19.21181869506836 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 19.487552642822266 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 19.57048797607422 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 19.376440048217773 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 19.439638137817383 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 19.440250396728516 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 19.474699020385742 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 19.5200138092041 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 19.47953987121582 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 19.47177505493164 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 19.46965980529785 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 19.471664428710938 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 19.5199031829834 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 19.483200073242188 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 19.48479652404785 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 19.456186294555664 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 19.46101188659668 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 19.468711853027344 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 19.473140716552734 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 19.476594924926758 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 19.58858871459961 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 19.457286834716797 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 19.4429988861084 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 19.477537155151367 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 19.46047592163086 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 19.5267276763916 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 19.535490036010742 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 19.509035110473633 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 19.484434127807617 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 19.448856353759766 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 19.45000457763672 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 19.442392349243164 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 19.47898292541504 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 19.48110008239746 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 19.464004516601562 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 19.48402976989746 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 19.46696662902832 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 19.47622299194336 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 19.478212356567383 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 19.462167739868164 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 19.504528045654297 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 19.54010581970215 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 19.50507164001465 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 19.469722747802734 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 19.4632625579834 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 19.463132858276367 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 19.491971969604492 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 19.493942260742188 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 19.45667839050293 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 19.482585906982422 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 19.519794464111328 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 19.509925842285156 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 19.496702194213867 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 19.489927291870117 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 19.5017147064209 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 19.51060676574707 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 19.496028900146484 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 19.479928970336914 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 19.516124725341797 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 19.48777961730957 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 19.4869327545166 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 19.496536254882812 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 19.506772994995117 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 19.469709396362305 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 19.469863891601562 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 19.49131965637207 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 19.4677734375 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 19.484737396240234 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 19.493677139282227 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 19.519874572753906 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 19.493881225585938 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 19.47350311279297 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 19.474157333374023 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 19.47359848022461 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 19.483802795410156 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 19.496023178100586 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 19.484371185302734 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 19.46565818786621 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 19.4865665435791 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 19.499576568603516 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 19.47869873046875 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 19.509387969970703 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 19.510726928710938 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 19.503625869750977 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 19.506746292114258 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 19.46870231628418 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 19.462268829345703 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 19.461973190307617 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 19.526168823242188 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 19.535205841064453 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 19.536561965942383 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 19.53476333618164 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 19.47120475769043 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 19.488510131835938 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 19.47323226928711 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 19.468942642211914 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 19.49609375 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 19.524961471557617 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 19.4998779296875 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 19.484949111938477 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 19.49622917175293 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 19.475801467895508 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 19.519641876220703 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 19.515230178833008 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 19.53011131286621 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 19.530941009521484 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 19.482032775878906 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 19.504337310791016 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 19.487232208251953 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 19.507156372070312 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 19.51081657409668 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 19.513593673706055 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 19.51580810546875 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 19.465791702270508 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 19.49074935913086 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 19.47392463684082 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 19.518247604370117 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 19.51885986328125 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 19.511573791503906 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 19.499195098876953 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 19.477203369140625 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 19.489513397216797 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 19.491050720214844 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 19.50477409362793 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 19.508039474487305 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 19.477190017700195 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 19.476572036743164 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 19.48077964782715 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 19.481081008911133 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 19.486663818359375 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 19.512361526489258 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 19.518190383911133 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 19.51508331298828 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 19.50885772705078 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 19.477657318115234 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 19.484378814697266 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 19.489442825317383 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 19.506614685058594 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 19.514867782592773 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 19.50662612915039 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 19.499897003173828 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 19.490856170654297 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 19.48927116394043 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 19.476970672607422 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 19.493453979492188 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 19.514175415039062 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 19.509248733520508 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 19.51805877685547 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 19.494977951049805 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 19.492599487304688 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 19.471961975097656 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 19.498228073120117 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 19.493980407714844 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 19.519845962524414 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 19.494834899902344 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 19.47719955444336 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 19.444910049438477 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 19.491037368774414 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 19.491878509521484 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 19.508132934570312 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 19.5018253326416 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 19.49302101135254 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 19.487306594848633 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 19.493871688842773 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 19.478553771972656 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 19.506284713745117 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 19.505695343017578 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 19.50352668762207 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 19.508237838745117 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 19.48948097229004 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 19.49532699584961 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 19.49121856689453 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 19.50826644897461 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 19.510791778564453 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 19.5156307220459 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 19.503643035888672 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 19.484806060791016 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 19.504318237304688 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 19.491697311401367 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 19.515148162841797 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 19.51980972290039 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 19.528732299804688 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 19.49665641784668 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 19.495193481445312 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 19.51213264465332 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 19.49224090576172 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 19.50908088684082 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 19.511070251464844 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 19.514253616333008 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 19.506927490234375 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 19.494062423706055 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 19.49497413635254 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 19.48941421508789 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 19.514169692993164 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 19.511842727661133 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 19.520992279052734 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 19.523235321044922 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 19.504486083984375 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 19.52942657470703 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 19.4958438873291 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 19.498933792114258 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 19.498571395874023 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 19.505666732788086 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 19.50213623046875 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 19.499130249023438 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 19.485383987426758 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 19.479473114013672 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 19.506093978881836 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 19.513151168823242 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 19.51413345336914 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 19.49407196044922 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 19.493881225585938 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 19.502792358398438 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 19.494672775268555 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 19.510894775390625 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 19.50594711303711 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 19.506105422973633 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 19.512788772583008 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 19.494699478149414 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 19.49968719482422 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 19.49970245361328 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 19.520475387573242 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 19.52568244934082 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 19.51067543029785 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 19.52897071838379 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 19.499069213867188 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 19.51035499572754 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 19.491474151611328 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 19.50830841064453 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 19.504838943481445 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 19.500476837158203 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 19.519521713256836 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 19.50344467163086 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 19.4842586517334 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 19.496850967407227 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 19.534894943237305 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 19.526702880859375 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 19.53070831298828 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 19.496931076049805 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 19.507713317871094 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 19.504192352294922 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 19.486316680908203 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 19.49445343017578 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 19.508638381958008 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 19.50665283203125 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 19.491432189941406 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 19.486644744873047 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 19.511674880981445 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 19.492361068725586 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 19.525545120239258 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 19.50737762451172 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 19.496601104736328 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 19.503101348876953 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 19.494136810302734 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 19.59165382385254 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 19.503955841064453 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 19.382104873657227 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 19.500560760498047 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 19.467426300048828 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 19.478727340698242 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 19.449296951293945 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 19.56968879699707 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 19.54319190979004 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 19.279272079467773 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 19.388381958007812 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 19.484058380126953 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 19.478015899658203 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 19.565359115600586 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 19.531606674194336 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 19.490131378173828 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 19.483184814453125 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 19.47614860534668 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 19.51128578186035 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 19.534624099731445 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 19.474390029907227 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 19.542152404785156 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 19.510950088500977 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 19.416841506958008 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 19.496475219726562 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 19.50637435913086 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 19.481637954711914 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 19.445091247558594 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 19.586151123046875 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 19.508968353271484 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 19.27217674255371 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 19.40050506591797 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 19.480304718017578 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 19.459875106811523 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 19.493974685668945 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 19.464126586914062 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 19.49640464782715 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 19.450157165527344 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 19.44194793701172 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 19.54353141784668 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 19.496633529663086 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 19.52387809753418 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 19.517139434814453 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 19.503475189208984 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 19.52500343322754 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 19.5211124420166 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 19.54664421081543 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 19.52132797241211 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 19.46940803527832 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 19.488317489624023 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 19.453163146972656 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 19.492231369018555 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 19.511438369750977 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 19.539953231811523 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 19.520410537719727 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 19.353490829467773 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 19.394378662109375 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 19.34039878845215 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 19.484745025634766 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 19.51390838623047 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 19.523807525634766 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 19.530256271362305 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 19.595563888549805 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 19.661178588867188 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 19.114301681518555 + } + ] + }, + { + "bit": 3, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.610965251922607 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.613927364349365 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.600709438323975 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.602231979370117 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.605029106140137 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.634901523590088 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.593055248260498 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.598849296569824 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.604278564453125 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.595339298248291 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.601174354553223 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.595224857330322 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.598481178283691 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.601424217224121 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.606837272644043 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.611329555511475 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.599499225616455 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.596702575683594 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.601352214813232 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.59116792678833 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.5995330810546875 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.547372341156006 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.593655586242676 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.596595287322998 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.597616195678711 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.602461338043213 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.601716041564941 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.60189151763916 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.608161449432373 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.608631610870361 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.6110520362854 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.608142375946045 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.603720188140869 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.6025848388671875 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.603219032287598 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.608180046081543 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.608264446258545 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.607038497924805 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.606386661529541 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.604930877685547 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.604173183441162 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.6059112548828125 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.609254360198975 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.608376979827881 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.608383655548096 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.6081719398498535 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.604726791381836 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.602941036224365 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.606283187866211 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.61045503616333 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.6099534034729 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.607869625091553 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.610583305358887 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.605042457580566 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.604157447814941 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.603970527648926 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.611319541931152 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.610701560974121 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.6101179122924805 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.613403797149658 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.610960006713867 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.608672618865967 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.608251094818115 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.609776020050049 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.609670639038086 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.6096038818359375 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.608339309692383 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.610261917114258 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.60866641998291 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.607597351074219 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.611359596252441 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.610268592834473 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.611976623535156 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.610629081726074 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.607296466827393 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.609368801116943 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.60718297958374 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.6080780029296875 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.609432697296143 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.60975456237793 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.611152172088623 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.607362747192383 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.6056318283081055 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.606940746307373 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.613192081451416 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.614726543426514 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.609540939331055 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.610897541046143 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.609951972961426 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.610222816467285 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.609403133392334 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.602160453796387 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.604036331176758 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.604907512664795 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.609195232391357 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.615489959716797 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.613883018493652 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.609699726104736 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.607491970062256 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.608156681060791 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.607515335083008 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.60283088684082 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.609900951385498 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.606540679931641 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.608771800994873 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.609640121459961 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.606873512268066 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.610374450683594 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.603282451629639 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.611891746520996 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.60896635055542 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.607772350311279 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.608265399932861 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.607812881469727 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.610090255737305 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.60677433013916 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.612811088562012 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.607217311859131 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.607560157775879 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.608071327209473 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.608471393585205 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.602667331695557 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.601504802703857 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.618419647216797 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.613097667694092 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.610074520111084 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.610043525695801 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.605715751647949 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.604620456695557 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.600820064544678 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.611429214477539 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.6248779296875 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.601630210876465 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.609295845031738 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.609548091888428 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.6022629737854 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.605740070343018 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.60654354095459 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.607891082763672 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.600700855255127 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.610763072967529 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.60955810546875 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.601879596710205 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.601922035217285 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.606722354888916 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.608865737915039 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.608625888824463 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.607686519622803 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.607641696929932 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.60383939743042 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.60275936126709 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.599990367889404 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.6136016845703125 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.604333877563477 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.603816032409668 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.604240417480469 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.610135555267334 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.596535682678223 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.603963851928711 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.616603374481201 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.6075263023376465 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.608180999755859 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.607590198516846 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.597535133361816 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.598446846008301 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.599501132965088 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.61431360244751 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.612841606140137 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.611669540405273 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.6080756187438965 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.6054253578186035 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.6019110679626465 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.602762699127197 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.610610485076904 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.607961177825928 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.610773086547852 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.6102776527404785 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.608789443969727 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.601539611816406 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.5962042808532715 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.603644371032715 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.596819877624512 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.608909606933594 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.608729839324951 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.611740589141846 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.602737903594971 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.59683084487915 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.6035356521606445 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.597143650054932 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.605772018432617 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.6075029373168945 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.606814384460449 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.606629848480225 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.606159210205078 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.605478763580322 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.605538845062256 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.6040778160095215 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.606455326080322 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.613137245178223 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.604691028594971 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.607691764831543 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.605686664581299 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.608092308044434 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.5995941162109375 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.605191230773926 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.6141133308410645 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.61409854888916 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.599437236785889 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.60438871383667 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.603968620300293 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.607117652893066 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.6093430519104 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.6092705726623535 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.610475540161133 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.605598449707031 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.609925746917725 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.602301120758057 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.607578277587891 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.606332302093506 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.6073408126831055 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.602787971496582 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.609049320220947 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.610398769378662 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.607889652252197 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 6.608952045440674 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 6.612752437591553 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 6.608219623565674 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 6.6122212409973145 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 6.609262943267822 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 6.609514236450195 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 6.602326393127441 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 6.6122212409973145 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 6.609766483306885 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 6.610201835632324 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 6.611051559448242 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 6.616575241088867 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 6.6149373054504395 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 6.617486953735352 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 6.623973369598389 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 6.617995262145996 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 6.618091583251953 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 6.617244720458984 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 6.590719699859619 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 6.590478420257568 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 6.612771511077881 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 6.610064506530762 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 6.608066082000732 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 6.607424736022949 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 6.612024307250977 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 6.6072540283203125 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 6.6087646484375 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 6.606405258178711 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 6.610234260559082 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 6.612975120544434 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 6.604930877685547 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 6.605311870574951 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 6.610178470611572 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 6.612173080444336 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 6.610250949859619 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 6.610225677490234 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 6.611224174499512 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 6.607433795928955 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 6.607816696166992 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 6.607506275177002 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 6.609640598297119 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 6.60554313659668 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 6.610720634460449 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 6.610398769378662 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 6.607883453369141 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 6.608519554138184 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 6.607645511627197 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 6.609138488769531 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 6.605021953582764 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 6.610119342803955 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 6.610754489898682 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 6.607244491577148 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 6.608085632324219 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 6.609450340270996 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 6.609521389007568 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 6.609223365783691 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 6.6114044189453125 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 6.610648155212402 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 6.610451698303223 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 6.6100335121154785 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 6.608219623565674 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 6.606410503387451 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 6.60915994644165 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 6.6104326248168945 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 6.610374450683594 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 6.610633850097656 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 6.607554912567139 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 6.609068870544434 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 6.608283042907715 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 6.608076572418213 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 6.610130310058594 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 6.610774040222168 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 6.61014986038208 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 6.610806941986084 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 6.608339786529541 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 6.610198974609375 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 6.6080827713012695 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 6.613234996795654 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 6.612401485443115 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 6.611141204833984 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 6.60797119140625 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 6.6072540283203125 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 6.606085777282715 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 6.606091022491455 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 6.611311912536621 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 6.609659671783447 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 6.611973762512207 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 6.610480785369873 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 6.609491348266602 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 6.605987548828125 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 6.603309154510498 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 6.610767841339111 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 6.610988140106201 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 6.609793186187744 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 6.609651565551758 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 6.6096038818359375 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 6.60996675491333 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 6.611071586608887 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 6.6105146408081055 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 6.61086368560791 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 6.6110148429870605 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 6.609590530395508 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 6.606700420379639 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 6.609886169433594 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 6.60374641418457 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 6.610881328582764 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 6.611591815948486 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 6.610919952392578 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 6.608335971832275 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 6.608531475067139 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 6.605847358703613 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 6.60731840133667 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 6.610959529876709 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 6.61090087890625 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 6.611051559448242 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 6.612232685089111 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 6.60933256149292 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 6.6090264320373535 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 6.607421398162842 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 6.611305236816406 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 6.611708641052246 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 6.610690593719482 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 6.6110124588012695 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 6.605898857116699 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 6.605777740478516 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 6.608826160430908 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 6.611020088195801 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 6.611152172088623 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 6.61083984375 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 6.6120405197143555 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 6.6119561195373535 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 6.608506679534912 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 6.606948375701904 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 6.611042022705078 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 6.610487461090088 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 6.610321998596191 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 6.611881256103516 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 6.608610153198242 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 6.603582859039307 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 6.609513759613037 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 6.611057758331299 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 6.609992980957031 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 6.611331939697266 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 6.608880043029785 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 6.605336666107178 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 6.61055326461792 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 6.609610557556152 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 6.610958099365234 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 6.611360549926758 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 6.610250949859619 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 6.609877109527588 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 6.605803966522217 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 6.608839988708496 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 6.605893135070801 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 6.610002040863037 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 6.610543251037598 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 6.610327243804932 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 6.610469818115234 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 6.607108116149902 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 6.608191967010498 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 6.60689640045166 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 6.610800743103027 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 6.610571384429932 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 6.610682487487793 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 6.610751152038574 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 6.609120845794678 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 6.6070170402526855 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 6.607326984405518 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 6.611005783081055 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 6.611740589141846 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 6.6109490394592285 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 6.6107916831970215 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 6.608665466308594 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 6.60537576675415 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 6.606816291809082 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 6.611730098724365 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 6.611361980438232 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 6.612204551696777 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 6.611855506896973 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 6.607600688934326 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 6.608104705810547 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 6.605233192443848 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 6.613338947296143 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 6.6132426261901855 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 6.612137794494629 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 6.610445022583008 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 6.608299255371094 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 6.610645771026611 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 6.607192516326904 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 6.611623764038086 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 6.611571788787842 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 6.612032890319824 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 6.610956192016602 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 6.610588073730469 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 6.60680627822876 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 6.607359409332275 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 6.610990047454834 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 6.611798286437988 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 6.610960006713867 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 6.611358642578125 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 6.608376979827881 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 6.609060764312744 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 6.608853340148926 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 6.611085414886475 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 6.610723495483398 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 6.610761642456055 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 6.610459327697754 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 6.6060285568237305 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 6.608320236206055 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 6.605485916137695 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 6.610965251922607 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 6.6112823486328125 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 6.610248565673828 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 6.610320568084717 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 6.607362270355225 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 6.607245445251465 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 6.606971740722656 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 6.610205173492432 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 6.610350131988525 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 6.611055850982666 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 6.610101699829102 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 6.6122355461120605 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 6.609659671783447 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 6.608277320861816 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 6.611231327056885 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 6.61082124710083 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 6.611568450927734 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 6.6116251945495605 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 6.605381011962891 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 6.607677459716797 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 6.608889102935791 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 6.6111159324646 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 6.612504959106445 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 6.609487533569336 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 6.61122989654541 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 6.607156276702881 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 6.6052117347717285 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 6.610816955566406 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 6.611365795135498 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 6.611324310302734 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 6.6104817390441895 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 6.6107916831970215 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 6.607341766357422 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 6.60771369934082 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 6.60565185546875 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 6.611379623413086 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 6.6099443435668945 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 6.610911846160889 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 6.610596179962158 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 6.609407424926758 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 6.60591983795166 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 6.607658386230469 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 6.6102728843688965 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 6.611935615539551 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 6.6093597412109375 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 6.6106181144714355 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 6.609010696411133 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 6.613559722900391 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 6.606659889221191 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 6.6123456954956055 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 6.612581729888916 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 6.611639499664307 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 6.610246181488037 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 6.609545707702637 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 6.611804962158203 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 6.6061577796936035 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 6.60767936706543 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 6.608976364135742 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 6.6106085777282715 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 6.60968542098999 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 6.606554985046387 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 6.619975566864014 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 6.60556697845459 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 6.611448764801025 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 6.6160736083984375 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 6.6088714599609375 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 6.6123857498168945 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 6.601993560791016 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 6.610810279846191 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 6.604674816131592 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 6.624056816101074 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 6.6206865310668945 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 6.611455917358398 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 6.613786697387695 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 6.604918003082275 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 6.609228134155273 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 6.613189220428467 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 6.614930152893066 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 6.613395690917969 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 6.610750675201416 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 6.607551574707031 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 6.605612277984619 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 6.612747669219971 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 6.610814094543457 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 6.613197326660156 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 6.615193843841553 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 6.617031097412109 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 6.609434127807617 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 6.603616237640381 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 6.605712413787842 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 6.6011857986450195 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 6.667774677276611 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 6.656827449798584 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 6.613356113433838 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 6.6181416511535645 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 6.587853908538818 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 6.605456829071045 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 6.610605239868164 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 6.614104270935059 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 6.619519233703613 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 6.610116958618164 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 6.603937149047852 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 6.602222919464111 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 6.61725378036499 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 6.628335952758789 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 6.618727207183838 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 6.61136531829834 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 6.614319801330566 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 6.611556529998779 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 6.6033477783203125 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 6.617676258087158 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 6.614666938781738 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 6.605957508087158 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 6.609961032867432 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 6.599791049957275 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 6.60671854019165 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 6.6028947830200195 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 6.619083881378174 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 6.6067118644714355 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 6.613232612609863 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 6.611635684967041 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 6.613293170928955 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 6.614716529846191 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 6.567111015319824 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 6.567124366760254 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 6.55609655380249 + } + ] + }, + { + "bit": 4, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.2345123291015625 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.234724998474121 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.234157085418701 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.233218669891357 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.233705043792725 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.234114646911621 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.233644008636475 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.233780860900879 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.23435640335083 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.233841896057129 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.232160568237305 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.233006000518799 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.23286771774292 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.233328819274902 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.232800006866455 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.234198093414307 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.233826160430908 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.233157634735107 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.233097553253174 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.23397970199585 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.232367515563965 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.233331680297852 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.234039306640625 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.233743667602539 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.233733654022217 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.2337236404418945 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.232530117034912 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.23220157623291 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.235121726989746 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.234509468078613 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.2340569496154785 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.234652042388916 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.2325544357299805 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.2324957847595215 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.233316898345947 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.23441219329834 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.234650611877441 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.234500885009766 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.234149932861328 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.23263692855835 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.232972145080566 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.233154773712158 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.23431396484375 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.23441743850708 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.2353410720825195 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.234920024871826 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.233592510223389 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.2336273193359375 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.233659744262695 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.234233856201172 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.234025955200195 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.23372745513916 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.2350687980651855 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.234110355377197 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.233841419219971 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.234145641326904 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.234405517578125 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.234660625457764 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.234351634979248 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.234689712524414 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.233930587768555 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.2346367835998535 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.234102249145508 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.234762191772461 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.234398365020752 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.234493732452393 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.234476089477539 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.234821319580078 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.235201358795166 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.235241413116455 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.234771728515625 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.2347002029418945 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.2342376708984375 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.234285831451416 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.233717441558838 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.233339786529541 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.233537673950195 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.23404598236084 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.234237194061279 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.234261512756348 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.234358787536621 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.233208656311035 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.2347941398620605 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.232874870300293 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.236099720001221 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.236778259277344 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.234464168548584 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.235544204711914 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.23420524597168 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.233524322509766 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.234278678894043 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.2330322265625 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.2324700355529785 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.234122276306152 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.2332634925842285 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.2357354164123535 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.23580265045166 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.235567092895508 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.23468542098999 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.2340545654296875 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.233821868896484 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.234126567840576 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.234757900238037 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.234597206115723 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.2351484298706055 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.233428001403809 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.234337329864502 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.235573768615723 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.234646320343018 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.2356390953063965 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.233880043029785 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.233695983886719 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.233951568603516 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.234522342681885 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.236603260040283 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.2330322265625 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.234145641326904 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.23305082321167 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.234026908874512 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.234928607940674 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.233876705169678 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.233252048492432 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.234087944030762 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.235874176025391 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.234248638153076 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.234160900115967 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.233937740325928 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.234970569610596 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.232289791107178 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.2310686111450195 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.232730388641357 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.234421253204346 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.2344746589660645 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.235067844390869 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.235286235809326 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.230980396270752 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.232321262359619 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.232966423034668 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.23435640335083 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.232054233551025 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.234854221343994 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.234326362609863 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.230791091918945 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.23288631439209 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.2334113121032715 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.235858917236328 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.232797622680664 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.23443078994751 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.234744548797607 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.230889797210693 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.229658603668213 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.232143878936768 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.23297643661499 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.235540390014648 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.231313705444336 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.2323784828186035 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.226524829864502 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.231322288513184 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.233663082122803 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.236146450042725 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.236469268798828 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.233809471130371 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.234821796417236 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.231022357940674 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.229543685913086 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.2323899269104 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.236203670501709 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.234793186187744 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.234516620635986 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.234582424163818 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.232020854949951 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.2335205078125 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.231547832489014 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.235422134399414 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.234365463256836 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.234100818634033 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.234595775604248 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.235886573791504 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.232084274291992 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.22908878326416 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.23003625869751 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.232272624969482 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.23447847366333 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.233940601348877 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.23209810256958 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.233062267303467 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.230153560638428 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.2295308113098145 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.2332987785339355 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.232873439788818 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.232874870300293 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.230172634124756 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.23158597946167 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.231833457946777 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.232179164886475 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.231100082397461 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.23306941986084 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.233465194702148 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.232446193695068 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.232796669006348 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.233041286468506 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.230914115905762 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.231895923614502 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.234025955200195 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.2328290939331055 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.233333110809326 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.233689785003662 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.2308244705200195 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.230295658111572 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.229841709136963 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.232892990112305 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.234932899475098 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.232147693634033 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.232789039611816 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.232070446014404 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.230454444885254 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.233349800109863 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.233524322509766 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.232391834259033 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.23266077041626 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.235385894775391 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.234039306640625 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.23341178894043 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.234606742858887 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.233999729156494 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.234026908874512 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.234447956085205 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.234677314758301 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.233428001403809 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.234281539916992 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.2346320152282715 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.2347517013549805 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.235706806182861 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.233458995819092 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.234771728515625 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.23320198059082 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.235432147979736 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.234115123748779 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.235730171203613 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.234742164611816 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.237746715545654 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.2333903312683105 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.226993560791016 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.231899738311768 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.235610485076904 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.235095977783203 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.233043670654297 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.232298851013184 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.233059406280518 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.233201503753662 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.234312534332275 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.234165668487549 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.2319560050964355 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.2337846755981445 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.2350664138793945 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.233238220214844 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.234273910522461 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.234323501586914 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.234874248504639 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.235265254974365 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.234281539916992 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.232420921325684 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.231349468231201 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.233377933502197 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.232769012451172 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.233180522918701 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.234742164611816 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.233700275421143 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.233139514923096 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.234516143798828 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.232480049133301 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.232585906982422 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.233560085296631 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.233889579772949 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.234524726867676 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.233362674713135 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.232733726501465 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.231705665588379 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.232172012329102 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.2324910163879395 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.2339558601379395 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.234157562255859 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.234625339508057 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.233978271484375 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.231105327606201 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.232676029205322 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.231435775756836 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.2345099449157715 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.234326362609863 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.233970642089844 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.234866619110107 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.2343831062316895 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.231293678283691 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.233078479766846 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.2340850830078125 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.234547138214111 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.234054088592529 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.234259128570557 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.232593059539795 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.229366779327393 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.233824729919434 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.235531330108643 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.234764575958252 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.235325336456299 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.234732151031494 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.232180118560791 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.234374523162842 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.231803894042969 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.234979152679443 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.234305381774902 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.234702110290527 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.235086441040039 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.2334794998168945 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.232836723327637 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.233471870422363 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.234421253204346 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.234501361846924 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.235209941864014 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.234043121337891 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.235034465789795 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.233092308044434 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.233808517456055 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.234393119812012 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.234309673309326 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.2336955070495605 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.234264373779297 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.232965469360352 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.232832431793213 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.231902122497559 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.234610080718994 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.234622001647949 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.235077381134033 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.233132839202881 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.232242584228516 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.232598781585693 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.231998443603516 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 5.234537601470947 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 5.234899044036865 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 5.2355427742004395 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 5.234500408172607 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 5.232170104980469 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 5.230401039123535 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 5.232637882232666 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 5.235369682312012 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 5.23506498336792 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 5.235203742980957 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 5.235304832458496 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 5.233371734619141 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 5.234703063964844 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 5.232436180114746 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 5.23474645614624 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 5.234766006469727 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 5.234916687011719 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 5.234516143798828 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 5.233786106109619 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 5.231561660766602 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 5.232646942138672 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 5.23439359664917 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 5.234737873077393 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 5.235279560089111 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 5.233673095703125 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 5.23191499710083 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 5.23336935043335 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 5.232218265533447 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 5.234370231628418 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 5.235547065734863 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 5.233729839324951 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 5.232700347900391 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 5.233526229858398 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 5.230531692504883 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 5.230605125427246 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 5.234608173370361 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 5.234910488128662 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 5.234467506408691 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 5.234745502471924 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 5.232284069061279 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 5.232086658477783 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 5.232590675354004 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 5.234510898590088 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 5.234560012817383 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 5.234216690063477 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 5.234250068664551 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 5.233346939086914 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 5.233125686645508 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 5.232518196105957 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 5.234626770019531 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 5.2348551750183105 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 5.233421802520752 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 5.234567165374756 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 5.233471870422363 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 5.234755992889404 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 5.2308430671691895 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 5.234835624694824 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 5.2351274490356445 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 5.235599994659424 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 5.234010696411133 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 5.232639312744141 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 5.233004570007324 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 5.231470584869385 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 5.2349371910095215 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 5.23486852645874 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 5.235139846801758 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 5.234344482421875 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 5.23277473449707 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 5.231750011444092 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 5.233283042907715 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 5.234564304351807 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 5.234676361083984 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 5.23460578918457 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 5.235108375549316 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 5.231813907623291 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 5.2320661544799805 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 5.233797550201416 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 5.234579563140869 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 5.2349772453308105 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 5.235086441040039 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 5.23441219329834 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 5.234868049621582 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 5.231256008148193 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 5.23306941986084 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 5.2344560623168945 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 5.234852313995361 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 5.23507022857666 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 5.235158920288086 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 5.234534740447998 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 5.233808517456055 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 5.233293056488037 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 5.234348773956299 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 5.234740734100342 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 5.233632564544678 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 5.234121799468994 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 5.232230186462402 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 5.232175350189209 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 5.231955051422119 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 5.234931468963623 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 5.234523773193359 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 5.235235214233398 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 5.234304904937744 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 5.233610153198242 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 5.234258651733398 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 5.2334818840026855 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 5.234897136688232 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 5.234832763671875 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 5.23502254486084 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 5.233776092529297 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 5.232062339782715 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 5.235300064086914 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 5.234251976013184 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 5.234887599945068 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 5.234685897827148 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 5.233683109283447 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 5.234724998474121 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 5.2342658042907715 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 5.234408378601074 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 5.2323150634765625 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 5.235152721405029 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 5.235324382781982 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 5.23569393157959 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 5.235170841217041 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 5.235362529754639 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 5.2333526611328125 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 5.231621265411377 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 5.234891414642334 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 5.234452247619629 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 5.2354536056518555 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 5.233847141265869 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 5.233639717102051 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 5.232280254364014 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 5.232845306396484 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 5.23409366607666 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 5.234690189361572 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 5.235032558441162 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 5.235254764556885 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 5.232213973999023 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 5.229640960693359 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 5.232580661773682 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 5.233621597290039 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 5.234900951385498 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 5.233545303344727 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 5.233783721923828 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 5.231960773468018 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 5.231238842010498 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 5.233548641204834 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 5.236264228820801 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 5.234499454498291 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 5.2330451011657715 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 5.233398914337158 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 5.2321295738220215 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 5.230000019073486 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 5.232165813446045 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 5.23175048828125 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 5.233632564544678 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 5.233738422393799 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 5.231320381164551 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 5.228599548339844 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 5.236026763916016 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 5.2309956550598145 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 5.233759880065918 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 5.235324859619141 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 5.230654716491699 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 5.235480785369873 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 5.224568843841553 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 5.23811674118042 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 5.229335784912109 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 5.2354865074157715 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 5.2351226806640625 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 5.2327561378479 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 5.233858585357666 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 5.230081081390381 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 5.234702110290527 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 5.233304500579834 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 5.23383903503418 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 5.23771333694458 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 5.232522010803223 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 5.230861663818359 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 5.231065273284912 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 5.2362565994262695 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 5.23635721206665 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 5.235204696655273 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 5.236952781677246 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 5.2306623458862305 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 5.232555866241455 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 5.224445343017578 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 5.233603000640869 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 5.228565216064453 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 5.248538970947266 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 5.257465839385986 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 5.234036922454834 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 5.237513065338135 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 5.222657203674316 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 5.237433910369873 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 5.2303996086120605 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 5.237361907958984 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 5.242830753326416 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 5.231778621673584 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 5.232173442840576 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 5.233608722686768 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 5.238428115844727 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 5.23715877532959 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 5.235054016113281 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 5.238511085510254 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 5.23075008392334 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 5.235110282897949 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 5.23193359375 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 5.2391252517700195 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 5.237936019897461 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 5.232160568237305 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 5.2313079833984375 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 5.231424331665039 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 5.230348110198975 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 5.226541519165039 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 5.2371416091918945 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 5.232949733734131 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 5.235978126525879 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 5.235952854156494 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 5.235163688659668 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 5.236321449279785 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 5.220906734466553 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 5.220452785491943 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 5.208448886871338 + } + ] + }, + { + "bit": 5, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.6655073165893555 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.665504455566406 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.665619373321533 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.6650071144104 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.664819240570068 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.664980888366699 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.665038585662842 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.66486930847168 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.664827346801758 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.664454460144043 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.665378570556641 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.664853572845459 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.664969444274902 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.664243221282959 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.6648969650268555 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.665228366851807 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.665253639221191 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.665019989013672 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.6650309562683105 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.664028644561768 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.664562225341797 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.664766311645508 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.665034294128418 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.665145397186279 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.664932727813721 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.664371967315674 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.664756774902344 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.664910316467285 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.66514253616333 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.665628910064697 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.66541051864624 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.66511869430542 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.665078639984131 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.664498805999756 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.664700508117676 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.665326118469238 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.665210723876953 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.6653547286987305 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.665182113647461 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.664698123931885 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.664976119995117 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.664767265319824 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.665154933929443 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.665235996246338 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.665261268615723 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.665374755859375 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.664705753326416 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.664244174957275 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.664604187011719 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.665416717529297 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.665435314178467 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.665378570556641 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.664909839630127 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.664949417114258 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.664863586425781 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.665218353271484 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.665038585662842 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.665305137634277 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.664840221405029 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.665126800537109 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.665531158447266 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.665694713592529 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.664478302001953 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.665415287017822 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.665126323699951 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.665405750274658 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.665410041809082 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.665249824523926 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.664897918701172 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.664681434631348 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.665022373199463 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.665184020996094 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.665555477142334 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.66488790512085 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.664875507354736 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.665134906768799 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.665485382080078 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.665637969970703 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.665319919586182 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.665454387664795 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.666015625 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.665270805358887 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.6648736000061035 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.665050506591797 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.666157245635986 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.666088581085205 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.666146278381348 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.665248394012451 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.665706634521484 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.665475368499756 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.664666652679443 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.664517402648926 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.664193153381348 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.66526985168457 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.664748191833496 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.66556453704834 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.665459632873535 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.665369987487793 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.664971828460693 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.664717197418213 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.665579795837402 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.664200305938721 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.665625095367432 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.664475440979004 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.665821075439453 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.665042400360107 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.665306091308594 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.6646552085876465 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.665188312530518 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.665626525878906 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.665194034576416 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.665378093719482 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.664877414703369 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.665034770965576 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.665307521820068 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.665508270263672 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.664505481719971 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.664792060852051 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.664665222167969 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.665444374084473 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.664744853973389 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.664719104766846 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.663960933685303 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.66579532623291 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.665729522705078 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.665706157684326 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.665189266204834 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.665946960449219 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.663860321044922 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.665213584899902 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.665022373199463 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.66519832611084 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.665474891662598 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.665506839752197 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.664802551269531 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.664239406585693 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.663527011871338 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.664829730987549 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.665592193603516 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.663973808288574 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.665517330169678 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.665220737457275 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.663949489593506 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.665380477905273 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.663967132568359 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.664861679077148 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.663562774658203 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.6655778884887695 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.665043354034424 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.663355827331543 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.6649627685546875 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.663822174072266 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.665021896362305 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.662964820861816 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.6638078689575195 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.664781093597412 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.663695812225342 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.663527965545654 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.663760185241699 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.665619850158691 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.66323184967041 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.6652960777282715 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.664915561676025 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.661856174468994 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.662756443023682 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.663376808166504 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.66595983505249 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.664151668548584 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.665241718292236 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.665205478668213 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.664018630981445 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.664093971252441 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.663083076477051 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.664903163909912 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.665316104888916 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.664636611938477 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.665095329284668 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.6634650230407715 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.663814067840576 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.663046360015869 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.664968013763428 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.663755893707275 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.665124893188477 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.665269374847412 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.665293216705322 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.663509845733643 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.663139820098877 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.665661334991455 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.66578483581543 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.664734363555908 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.664604187011719 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.663484573364258 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.663875102996826 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.663908004760742 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.66441011428833 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.665450572967529 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.664433002471924 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.665175437927246 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.664592266082764 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.664069652557373 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.665815353393555 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.66438102722168 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.663556098937988 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.664822578430176 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.665360927581787 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.664493560791016 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.664973258972168 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.665003776550293 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.665192127227783 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.6637864112854 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.664836406707764 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.665657997131348 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.663300037384033 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.666285991668701 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.664658069610596 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.663644790649414 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.6644511222839355 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.665018081665039 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.665191173553467 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.663053035736084 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.663641929626465 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.664203643798828 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.663651466369629 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.664823532104492 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.665308475494385 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.665494918823242 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.665846347808838 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.664371967315674 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.664455413818359 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.663760185241699 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.663661003112793 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.666059494018555 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.665542125701904 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.665562152862549 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.664597511291504 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.6646881103515625 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.666507720947266 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.665203094482422 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.6657891273498535 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.665665626525879 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.665768146514893 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.66484260559082 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.6630144119262695 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.663924217224121 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.662503242492676 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.6647257804870605 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.66500997543335 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.664113521575928 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.66552734375 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.665042400360107 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.665247917175293 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.664991855621338 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.664111614227295 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.666409492492676 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.664076328277588 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.663654804229736 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.665213584899902 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.66404390335083 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.6653289794921875 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.665143013000488 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.66559362411499 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.665121555328369 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.664936065673828 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.665611743927002 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.664310932159424 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.663890361785889 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.665088653564453 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.66539192199707 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.665213108062744 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.665521144866943 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.665060043334961 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.663333415985107 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.664489269256592 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.66522741317749 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.665306091308594 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.664521217346191 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.663881778717041 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.664220809936523 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.664584636688232 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.66273307800293 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.665278434753418 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.6647491455078125 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.665560245513916 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.664569854736328 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.66558837890625 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.664792537689209 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.664320945739746 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.665289878845215 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.6651506423950195 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.66497802734375 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.664148330688477 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.663877010345459 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.6645379066467285 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.66473913192749 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.665319919586182 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.665390491485596 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.66485595703125 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.6650872230529785 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.663815975189209 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.664499759674072 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.6632537841796875 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.66625452041626 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.665685653686523 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.665185451507568 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.665889739990234 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.664581775665283 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.665270805358887 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.664762496948242 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.6650285720825195 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.665415287017822 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.665298938751221 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.6653032302856445 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.662900924682617 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.663792610168457 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.664785861968994 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.665012359619141 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.665361404418945 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.665088653564453 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.665046691894531 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.664462089538574 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.663942337036133 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.664401531219482 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.665251731872559 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.665316104888916 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.665133953094482 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.6652116775512695 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.664698600769043 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.664539813995361 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.6624674797058105 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.665266036987305 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.665664196014404 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.665259838104248 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.66533899307251 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.663412570953369 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.664287567138672 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.664113998413086 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.665581703186035 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.665373802185059 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.665378093719482 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.665307521820068 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.6647419929504395 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.664701461791992 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.6643900871276855 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.665534973144531 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.665668964385986 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.665752410888672 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.665356159210205 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.6651291847229 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.662054538726807 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.665519714355469 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.6655192375183105 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.665365219116211 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.664807319641113 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.665156841278076 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.664097309112549 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.664459228515625 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.664073944091797 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.665270805358887 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.66522741317749 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.664399147033691 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.664974212646484 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.663904190063477 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.663470268249512 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.663531303405762 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.665431976318359 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.664881706237793 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.665143966674805 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.664575099945068 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.66298246383667 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.663765907287598 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.664362907409668 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.665672302246094 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.665274620056152 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.665063858032227 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.6646013259887695 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.6641974449157715 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.664366722106934 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.6643967628479 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.665305137634277 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.665395736694336 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.664940357208252 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.665337562561035 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.664596080780029 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.663337707519531 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.664738178253174 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.665607929229736 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.665323734283447 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.66611909866333 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.66582727432251 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.663705825805664 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.665405750274658 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.664392948150635 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.66562557220459 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.665279865264893 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.665122985839844 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.665147304534912 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.663414478302002 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.665101051330566 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.665306091308594 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.665545463562012 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.665497303009033 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.665006160736084 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.665127277374268 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.6635894775390625 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.665513515472412 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.6641364097595215 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.665780067443848 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.665398597717285 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.665605545043945 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.664944171905518 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.664781093597412 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.66452169418335 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.6631598472595215 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.665143966674805 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.665184020996094 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.665943622589111 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.6652302742004395 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.665799617767334 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.664483547210693 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.664106369018555 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.665318489074707 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.66541051864624 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.665618896484375 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.665462970733643 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.664039134979248 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.664011001586914 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.6644206047058105 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.66533088684082 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.665272235870361 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.665337085723877 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.665445327758789 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.664226055145264 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.664031028747559 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.664669513702393 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.665210723876953 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.665092468261719 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.665468692779541 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.664906978607178 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.66336727142334 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.662800312042236 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.6646904945373535 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.665519714355469 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.665204048156738 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.665411949157715 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.665178298950195 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.664571285247803 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.6639227867126465 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.663233757019043 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.665163993835449 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.6651763916015625 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.665029525756836 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.665140628814697 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.664763450622559 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.664151668548584 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.664997100830078 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.665436744689941 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.665694713592529 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.6651129722595215 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.6651811599731445 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.6635918617248535 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.663746356964111 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.663158416748047 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.665414333343506 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.665350914001465 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.6646037101745605 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.664793491363525 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.663970947265625 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.663045406341553 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.664334774017334 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.665402889251709 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.664951801300049 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.665016174316406 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.66550874710083 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.664483070373535 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.665014743804932 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.6628875732421875 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.66483736038208 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.66483211517334 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.665087699890137 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.6645073890686035 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.6653900146484375 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.664479732513428 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.664012432098389 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.665674209594727 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.665370941162109 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.665487289428711 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.665913105010986 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.66387414932251 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.664021968841553 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.6659393310546875 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.665036201477051 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.664652347564697 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.664311408996582 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.664768695831299 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.663261413574219 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.664990425109863 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.662185192108154 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.665416717529297 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.664738178253174 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.662887096405029 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.6640095710754395 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.659953594207764 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.666911602020264 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.662683963775635 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.665201663970947 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.666404724121094 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.662695407867432 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.664716720581055 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.66165018081665 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.666430950164795 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.667966365814209 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.663692474365234 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.666223049163818 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.66098690032959 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.662970542907715 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.661515235900879 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.663369178771973 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.664974212646484 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.664774417877197 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.666684627532959 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.664477825164795 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.664078712463379 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.659803867340088 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.6645188331604 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.662154197692871 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.679135799407959 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.67443323135376 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.661420822143555 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.664954662322998 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.65712833404541 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.664905548095703 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.665449142456055 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.666285037994385 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.66933012008667 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.663830280303955 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.661777496337891 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.664289951324463 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.667464733123779 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.668898105621338 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.665108680725098 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.666075229644775 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.665311336517334 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.6650495529174805 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.663914203643799 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.667708396911621 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.663498401641846 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.664215564727783 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.663644790649414 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.661942958831787 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.664433002471924 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.661923885345459 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.667248725891113 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.665548801422119 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.665587902069092 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.666409492492676 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.665066242218018 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.665439128875732 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.6574811935424805 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.659273624420166 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.65089225769043 + } + ] + }, + { + "bit": 6, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.355562210083008 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.355896472930908 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.355480194091797 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.355719566345215 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.35573148727417 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.355556964874268 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.355371952056885 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.355735778808594 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.355280876159668 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.354884147644043 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.355120658874512 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.355138778686523 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.355637073516846 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.355834007263184 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.3556647300720215 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.355702877044678 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.355570316314697 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.355569362640381 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.355428218841553 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.355766773223877 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.355376243591309 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.355570316314697 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.3558030128479 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.35547399520874 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.35526180267334 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.355328559875488 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.355765342712402 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.355257511138916 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.355551719665527 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.355537414550781 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.355935096740723 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.355587959289551 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.355338096618652 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.355471611022949 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.355224609375 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.355616569519043 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.356034755706787 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.356287956237793 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.355690956115723 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.355586051940918 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.355877876281738 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.355338096618652 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.355698108673096 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.355719566345215 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.35584020614624 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.355546474456787 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.35546875 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.35551643371582 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.355640411376953 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.355860233306885 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.3556623458862305 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.355686664581299 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.355868816375732 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.355663776397705 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.356086730957031 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.355193138122559 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.355693817138672 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.355622291564941 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.355868339538574 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.355843544006348 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.355584621429443 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.3556013107299805 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.355781555175781 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.355772495269775 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.35594367980957 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.356057167053223 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.355650424957275 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.355767726898193 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.355947017669678 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.355637550354004 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.355987071990967 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.355611324310303 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.355556964874268 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.355672359466553 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.355754852294922 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.355989456176758 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.355461597442627 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.355546474456787 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.355717182159424 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.355693817138672 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.355537414550781 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.354942321777344 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.355112075805664 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.355370044708252 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.355976581573486 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.356112003326416 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.355564117431641 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.356232166290283 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.35600471496582 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.355878829956055 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.3558197021484375 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.355379104614258 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.355368614196777 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.35496187210083 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.3554816246032715 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.356141567230225 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.355611324310303 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.355587482452393 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.355844497680664 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.356435298919678 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.355764389038086 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.356371879577637 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.355824947357178 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.355583667755127 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.356249809265137 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.355869293212891 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.35568904876709 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.356285095214844 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.3556694984436035 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.356239318847656 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.355868816375732 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.355417728424072 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.355751991271973 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.355431079864502 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.355958938598633 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.356205940246582 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.356099605560303 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.355894565582275 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.355574607849121 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.355724334716797 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.355461597442627 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.354720115661621 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.3564252853393555 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.356345176696777 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.355671405792236 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.355649948120117 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.355349063873291 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.356031894683838 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.355683326721191 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.3556647300720215 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.35576057434082 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.356366157531738 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.354860305786133 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.355682373046875 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.356362819671631 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.354701519012451 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.3550262451171875 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.354823589324951 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.355537414550781 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.355384349822998 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.356169700622559 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.355881690979004 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.354907035827637 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.355086326599121 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.355803966522217 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.355870723724365 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.355869770050049 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.355818271636963 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.355682849884033 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.353882312774658 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.35408353805542 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.355315208435059 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.3550825119018555 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.355813503265381 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.354899883270264 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.355071067810059 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.354087829589844 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.354518890380859 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.355432033538818 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.355462551116943 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.35622501373291 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.355467319488525 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.35564661026001 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.354503154754639 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.355292320251465 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.355210304260254 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.35629940032959 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.3562846183776855 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.35563325881958 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.356189727783203 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.354989051818848 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.355015754699707 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.35567045211792 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.355556488037109 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.355854034423828 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.355859279632568 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.356005668640137 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.35577392578125 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.355982303619385 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.354410648345947 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.354712963104248 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.3551740646362305 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.355879783630371 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.355863571166992 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.355835437774658 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.355021953582764 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.355144500732422 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.354950428009033 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.354362964630127 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.3552398681640625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.355448246002197 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.355084419250488 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.355618953704834 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.356122016906738 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.355327606201172 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.355220794677734 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.355964660644531 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.355658531188965 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.355599403381348 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.356194496154785 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.355335712432861 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.35493803024292 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.354831695556641 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.355740070343018 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.3559889793396 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.355782985687256 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.354889392852783 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.355839252471924 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.354660511016846 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.355704307556152 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.355529308319092 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.355399131774902 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.356393814086914 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.355093002319336 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.354191303253174 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.355735778808594 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.354805946350098 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.355502605438232 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.355373382568359 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.355175018310547 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.355356216430664 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.355661869049072 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.355014801025391 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.355345249176025 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.355691909790039 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.35574197769165 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.355047702789307 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.355441093444824 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.35519552230835 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.355881690979004 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.355433464050293 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.3561882972717285 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.356257915496826 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.355156421661377 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.355067729949951 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.355627059936523 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.355330944061279 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.355467319488525 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.355842590332031 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.356169700622559 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.355536460876465 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.356077194213867 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.354672431945801 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.3540802001953125 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.356326103210449 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.355993747711182 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.355332374572754 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.355515480041504 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.355240821838379 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.355099201202393 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.357003688812256 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.3554205894470215 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.355804920196533 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.3552680015563965 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.35600471496582 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.355391979217529 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.355557441711426 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.356064796447754 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.356179237365723 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.3552398681640625 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.355354309082031 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.354609966278076 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.355160713195801 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.354834079742432 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.355501651763916 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.355355739593506 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.355957508087158 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.355969429016113 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.355790615081787 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.354856014251709 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.355351448059082 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.355788707733154 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.35467004776001 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.35590124130249 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.355982303619385 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.35548734664917 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.356230735778809 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.3555192947387695 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.3545002937316895 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.35688591003418 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.355804443359375 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.355692386627197 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.356101989746094 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.355762481689453 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.355436325073242 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.3551344871521 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.35491418838501 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.355913162231445 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.3557209968566895 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.355680465698242 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.355853080749512 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.354992389678955 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.3547492027282715 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.35429573059082 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.355978012084961 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.355775833129883 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.3561625480651855 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.355686664581299 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.355618476867676 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.355159282684326 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.355196475982666 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.355795383453369 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.3559370040893555 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.355705261230469 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.355132102966309 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.354355812072754 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.354970932006836 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.354159832000732 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.355839252471924 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.355709552764893 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.356173515319824 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.355927467346191 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.356180667877197 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.355119705200195 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.353692054748535 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.355865001678467 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.355958461761475 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.355676174163818 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.35569953918457 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.3550519943237305 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.355580806732178 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.354513645172119 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.355765342712402 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.355814456939697 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.355747699737549 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.355411052703857 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.354855537414551 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.355501651763916 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.355651378631592 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.356142997741699 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.356025695800781 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.356006622314453 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.355302333831787 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.354902744293213 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.354768753051758 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.355342864990234 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.356010437011719 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.356019973754883 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.355900287628174 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.356285095214844 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.354726314544678 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.355075836181641 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.354891300201416 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.356268405914307 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.356346130371094 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.355813503265381 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.355774879455566 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.356076240539551 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.355154037475586 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.355223178863525 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.355862617492676 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.355975151062012 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.35609245300293 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.355419158935547 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.355242729187012 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.35593843460083 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.354815483093262 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.355800628662109 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.355831146240234 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.35658597946167 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.3561110496521 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.35499382019043 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.355608940124512 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.355106353759766 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.355659484863281 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.355246067047119 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.356191158294678 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.356122970581055 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.355467319488525 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.355700969696045 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.35402250289917 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.355969429016113 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.35579252243042 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.356031894683838 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.355697154998779 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.355693340301514 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.354626655578613 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.354152202606201 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.355828762054443 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.35575008392334 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.355990886688232 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.355580806732178 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.355683326721191 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.355643272399902 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.354675769805908 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.355966091156006 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.356016159057617 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.356001377105713 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.35599946975708 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.355338096618652 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.355626106262207 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.35482931137085 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.356074810028076 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.356008529663086 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.355992794036865 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.35603666305542 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.355488300323486 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.353843688964844 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.354175090789795 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.356052398681641 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.3560991287231445 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.355766773223877 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.355842590332031 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.355144023895264 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.355351448059082 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.355104923248291 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.355739116668701 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.355917930603027 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.355733394622803 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.355678558349609 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.354580879211426 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.355243682861328 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.355866432189941 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.355971336364746 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.356259822845459 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.355636119842529 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.356064796447754 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.3553080558776855 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.355198860168457 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.354936122894287 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.355996608734131 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.356076240539551 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.355663776397705 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.355818271636963 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.355708599090576 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.355982303619385 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.355335712432861 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.355894565582275 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.355897426605225 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.356049537658691 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.355705738067627 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.3544392585754395 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.355593204498291 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.354867458343506 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.355922698974609 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.35603666305542 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.355861186981201 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.355762481689453 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.354558944702148 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.356331825256348 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.355504035949707 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.35598611831665 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.356205940246582 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.355380535125732 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.355693340301514 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.355643272399902 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.355861186981201 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.3552985191345215 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.356025695800781 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.355836391448975 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.355877876281738 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.355867385864258 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.354872703552246 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.35476016998291 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.354377746582031 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.356138229370117 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.356049060821533 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.356034755706787 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.356067657470703 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.356616973876953 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.355587959289551 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.35610818862915 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.35601282119751 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.355925559997559 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.355814456939697 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.3561296463012695 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.355546474456787 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.354551315307617 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.355598449707031 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.355828762054443 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.356040954589844 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.356282711029053 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.355959892272949 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.355794429779053 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.3561015129089355 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.355836391448975 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.355750560760498 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.355615139007568 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.35567569732666 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.355813026428223 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.354090690612793 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.355367660522461 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.355047702789307 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.356482982635498 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.356010437011719 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.355382442474365 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.355786323547363 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.355812072753906 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.354339122772217 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.3534836769104 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.355617046356201 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.355655193328857 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.356637954711914 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.354766845703125 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.355154037475586 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.355256080627441 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.354639053344727 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.3555192947387695 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.356082916259766 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.353912353515625 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.356444358825684 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.352972030639648 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.3580121994018555 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.355804443359375 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.355471611022949 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.35628604888916 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.355481147766113 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.355224132537842 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.354009628295898 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.356666088104248 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.355778694152832 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.356234073638916 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.356801986694336 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.355807781219482 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.354580402374268 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.355088710784912 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.355554103851318 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.3553056716918945 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.3565993309021 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.357131481170654 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.353501319885254 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.355881690979004 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.35256814956665 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.355405330657959 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.3532915115356445 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.358588218688965 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.361457347869873 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.3508524894714355 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.355075359344482 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.352880477905273 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.3574676513671875 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.35537576675415 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.356354236602783 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.35780143737793 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.354249954223633 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.354564189910889 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.354030609130859 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.357242584228516 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.357632637023926 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.356608867645264 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.356379985809326 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.354945182800293 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.354946613311768 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.354637145996094 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.356090545654297 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.359558582305908 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.355043888092041 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.355668544769287 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.355313777923584 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.355011940002441 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.354499816894531 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.3571882247924805 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.356264114379883 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.357175350189209 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.356358051300049 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.356578826904297 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.3565850257873535 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.352389812469482 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.353078365325928 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.350094318389893 + } + ] + }, + { + "bit": 7, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.180251121520996 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.180264949798584 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.180347442626953 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.180107593536377 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.180171489715576 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.18009090423584 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.179759979248047 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.180092811584473 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.17995023727417 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.18047571182251 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.180269718170166 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.180074691772461 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.179775714874268 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.179629802703857 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.179933071136475 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.180018424987793 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.180083751678467 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.180371284484863 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.179805755615234 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.179976940155029 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.180185317993164 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.179960250854492 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.179900646209717 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.18039608001709 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.1800994873046875 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.180078506469727 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.17971134185791 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.179464340209961 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.1799163818359375 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.180119514465332 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.179723739624023 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.180084705352783 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.179975986480713 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.180250644683838 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.179870128631592 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.180272579193115 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.180336952209473 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.179895401000977 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.180152893066406 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.179555892944336 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.179998874664307 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.179994106292725 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.180091857910156 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.179929256439209 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.180466651916504 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.1802144050598145 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.18002462387085 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.18024206161499 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.180163383483887 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.179984092712402 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.1803765296936035 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.180015563964844 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.180239677429199 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.180257320404053 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.179989814758301 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.180111408233643 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.180191993713379 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.180267810821533 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.180310249328613 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.180292129516602 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.180240154266357 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.180175304412842 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.18017053604126 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.180305004119873 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.180087566375732 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.180346488952637 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.180120468139648 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.180184841156006 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.180197238922119 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.179710388183594 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.18040657043457 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.180072784423828 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.180367469787598 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.180189609527588 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.180493354797363 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.17978572845459 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.180671691894531 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.179986953735352 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.1802778244018555 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.1801276206970215 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.180054664611816 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.179873943328857 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.179744720458984 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.180096626281738 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.180229187011719 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.180322647094727 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.180479049682617 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.180459976196289 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.180062770843506 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.179904937744141 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.1799774169921875 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.179820537567139 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.17970609664917 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.180184364318848 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.180466651916504 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.18009090423584 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.180485248565674 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.180627346038818 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.180584907531738 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.180142402648926 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.180005073547363 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.180076599121094 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.18008279800415 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.18019437789917 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.179937839508057 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.180262088775635 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.180352210998535 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.180208206176758 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.179987907409668 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.180189609527588 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.1802449226379395 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.180202007293701 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.180108547210693 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.180014133453369 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.180116653442383 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.180042743682861 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.180357456207275 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.1801605224609375 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.180004119873047 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.180285930633545 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.180161476135254 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.180589199066162 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.179884433746338 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.180558204650879 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.180691719055176 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.179794788360596 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.180177688598633 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.180504322052002 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.179150581359863 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.179901599884033 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.179927349090576 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.179858684539795 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.180400371551514 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.180167198181152 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.180434226989746 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.180047988891602 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.179980278015137 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.180014133453369 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.180651664733887 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.179864406585693 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.180064678192139 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.180344104766846 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.179937839508057 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.180360794067383 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.179714679718018 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.1804728507995605 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.180218696594238 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.1802473068237305 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.179989337921143 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.179931163787842 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.180023193359375 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.1799092292785645 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.180619716644287 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.1798176765441895 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.179368019104004 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.1798014640808105 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.179704666137695 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.179358005523682 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.179568290710449 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.180233001708984 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.179779052734375 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.180148601531982 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.180049896240234 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.179671764373779 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.179158687591553 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.179431915283203 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.180505275726318 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.179690361022949 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.179896354675293 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.180171966552734 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.180027961730957 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.1800007820129395 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.179953098297119 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.1800618171691895 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.180272102355957 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.1800079345703125 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.180243968963623 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.180057048797607 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.179375171661377 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.179415702819824 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.179769992828369 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.1793131828308105 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.180357456207275 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.1799163818359375 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.180169582366943 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.180083751678467 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.180075168609619 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.1799702644348145 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.179744720458984 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.180049896240234 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.179964542388916 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.179836273193359 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.179471969604492 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.179225444793701 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.180019378662109 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.1805500984191895 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.17986536026001 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.179967880249023 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.179685592651367 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.179441928863525 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.1799726486206055 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.179561614990234 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.179181098937988 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.1798505783081055 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.179830074310303 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.180068492889404 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.180368900299072 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.179732799530029 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.180238246917725 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.179802417755127 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.180095195770264 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.180125713348389 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.17967414855957 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.180072784423828 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.1799492835998535 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.179896831512451 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.179780960083008 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.179951190948486 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.180168628692627 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.180107116699219 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.179633140563965 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.180246353149414 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.180115222930908 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.1806159019470215 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.179921627044678 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.180131435394287 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.180018901824951 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.1801066398620605 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.179713726043701 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.179514408111572 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.179650783538818 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.180303573608398 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.180324554443359 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.180068492889404 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.179871082305908 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.180024147033691 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.180307865142822 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.180599689483643 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.180569171905518 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.180198669433594 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.180697441101074 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.179740905761719 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.179301738739014 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.180243492126465 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.179214000701904 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.180213451385498 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.179915428161621 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.179390907287598 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.180181980133057 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.180386543273926 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.18009090423584 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.180059432983398 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.179972171783447 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.180710792541504 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.179355144500732 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.179769039154053 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.180148601531982 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.179766654968262 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.1788482666015625 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.179964065551758 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.1801862716674805 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.179934024810791 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.179439067840576 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.180269718170166 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.180403709411621 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.179980754852295 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.180017471313477 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.180131435394287 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.179832935333252 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.180627822875977 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.180261611938477 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.1793670654296875 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.179657459259033 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.179880142211914 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.179951190948486 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.179905414581299 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.1795573234558105 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.1791815757751465 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.180328845977783 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.179967880249023 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.180181980133057 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.1801018714904785 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.1800713539123535 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.17991828918457 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.179146766662598 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.18001651763916 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.180064678192139 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.180136680603027 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.1803717613220215 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.180275917053223 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.179325580596924 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.180356502532959 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.180373668670654 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.179330348968506 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.180215835571289 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.180167198181152 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.1800336837768555 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.180273056030273 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.179003715515137 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.179941177368164 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.179037570953369 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.18042516708374 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.180124759674072 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.180248737335205 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.180038928985596 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.180044651031494 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.179357051849365 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.180027008056641 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.180284023284912 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.180202484130859 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.180023193359375 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.180059432983398 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.17873477935791 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.179643630981445 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.179362773895264 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.180233478546143 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.180084705352783 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.180094242095947 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.180169582366943 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.1797966957092285 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.180121421813965 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.179210662841797 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.180198669433594 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.180130481719971 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.180187225341797 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.179904937744141 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.179839611053467 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.17988920211792 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.179201602935791 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.180189609527588 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.1801862716674805 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.180145740509033 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.180239200592041 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.179783821105957 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.179884433746338 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.179040908813477 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.18024206161499 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.180355072021484 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.180171012878418 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.1800642013549805 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.17997932434082 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.18071985244751 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.180321216583252 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.180402755737305 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.180564880371094 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.180204391479492 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.180610656738281 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.179281234741211 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.179430961608887 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.179091453552246 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.180151462554932 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.18010950088501 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.180049896240234 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.1799116134643555 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.18032693862915 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.179803848266602 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.180230140686035 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.180103302001953 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.180310249328613 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.179945468902588 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.179590225219727 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.179657459259033 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.179464817047119 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.180278778076172 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.18019437789917 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.1807637214660645 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.179773330688477 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.1800456047058105 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.179588317871094 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.179660797119141 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.179416656494141 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.180365562438965 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.180293083190918 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.180034160614014 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.17992639541626 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.179598808288574 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.1796674728393555 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.179881572723389 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.18017053604126 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.180221080780029 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.180185317993164 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.1798882484436035 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.179758548736572 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.179561614990234 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.17965030670166 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.180449485778809 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.180185794830322 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.180290222167969 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.179602146148682 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.1796956062316895 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.179296016693115 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.179382801055908 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.180422782897949 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.180315971374512 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.180485248565674 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.180149555206299 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.179200649261475 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.180222511291504 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.180099964141846 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.180340766906738 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.180255889892578 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.180446624755859 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.180158615112305 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.1798577308654785 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.179254531860352 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.179905891418457 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.180285930633545 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.180271148681641 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.180243968963623 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.180604934692383 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.180196285247803 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.180025577545166 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.178970813751221 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.180310249328613 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.180309295654297 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.180315971374512 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.180017948150635 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.180288791656494 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.179750442504883 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.179869174957275 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.180307865142822 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.1802849769592285 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.180526256561279 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.180411338806152 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.17950439453125 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.179659843444824 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.179563522338867 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.180149555206299 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.180274963378906 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.180028915405273 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.180074691772461 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.179991722106934 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.179462909698486 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.179798603057861 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.18021821975708 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.180264949798584 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.180262565612793 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.180228233337402 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.179379463195801 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.179126262664795 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.1791558265686035 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.1802778244018555 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.180303573608398 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.180161952972412 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.18035364151001 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.178980827331543 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.180257320404053 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.1800642013549805 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.18015193939209 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.180190086364746 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.179696083068848 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.180229187011719 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.1796369552612305 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.1804633140563965 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.180180549621582 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.180234909057617 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.180323600769043 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.180081844329834 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.1802520751953125 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.179247856140137 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.179253101348877 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.1796159744262695 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.180308818817139 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.180206298828125 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.180373191833496 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.1799468994140625 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.179765701293945 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.178491592407227 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.179782390594482 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.180240154266357 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.180150032043457 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.180192947387695 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.180070877075195 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.179539203643799 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.179596424102783 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.179243564605713 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.180055141448975 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.1802473068237305 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.180349826812744 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.179920196533203 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.180007457733154 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.179443836212158 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.179694652557373 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.180471420288086 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.18007230758667 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.180672645568848 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.180449485778809 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.179455757141113 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.179158687591553 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.180301189422607 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.179813861846924 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.179896354675293 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.179513454437256 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.179943084716797 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.1793389320373535 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.180230140686035 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.1798295974731445 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.18021821975708 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.18017053604126 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.178831100463867 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.180220127105713 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.1779985427856445 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.180990695953369 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.180047035217285 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.1798095703125 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.179862976074219 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.178549289703369 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.179997444152832 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.179103851318359 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.180233955383301 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.179183483123779 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.179676532745361 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.18091344833374 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.177848815917969 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.179121971130371 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.179103851318359 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.18081521987915 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.180052280426025 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.180138111114502 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.180031776428223 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.179839611053467 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.179291725158691 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.177758693695068 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.180967807769775 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.178497314453125 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.185445785522461 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.183468818664551 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.179610729217529 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.180448532104492 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.176945686340332 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.180343151092529 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.182572841644287 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.18080997467041 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.1825690269470215 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.180227279663086 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.17899227142334 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.1801862716674805 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.180619716644287 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.179901599884033 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.180073261260986 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.180687427520752 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.179469585418701 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.180363178253174 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.179872512817383 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.179996013641357 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.179973125457764 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.1801323890686035 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.179903984069824 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.178921222686768 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.179637432098389 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.178989887237549 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.181887626647949 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.179647445678711 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.1802592277526855 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.180866241455078 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.180253028869629 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.180315971374512 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.17767858505249 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.178481101989746 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.17279577255249 + } + ] + }, + { + "bit": 8, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.072144985198975 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.072070598602295 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.071688652038574 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.072108745574951 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.072051525115967 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.072169780731201 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.072389602661133 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.0721282958984375 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.071841239929199 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.07172155380249 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.071599960327148 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.0718302726745605 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.072105407714844 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.0720438957214355 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.072086334228516 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.071993350982666 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.072240829467773 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.071586608886719 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.072144508361816 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.071859836578369 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.0718092918396 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.072115421295166 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.072122097015381 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.072278022766113 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.0716352462768555 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.071537494659424 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.071675777435303 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.072021007537842 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.072104454040527 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.072256565093994 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.072009563446045 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.072253227233887 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.071634292602539 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.071962833404541 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.072053909301758 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.072152614593506 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.072075843811035 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.072328090667725 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.072037696838379 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.072064399719238 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.072083950042725 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.0722455978393555 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.072117805480957 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.072004795074463 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.072171688079834 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.072185516357422 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.07177734375 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.072059154510498 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.071977615356445 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.072073936462402 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.071974754333496 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.071977138519287 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.072296142578125 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.072248458862305 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.072389125823975 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.07159948348999 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.072277545928955 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.07197380065918 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.072356224060059 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.0719804763793945 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.071988105773926 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.072163105010986 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.072235107421875 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.07211446762085 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.071805953979492 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.0719146728515625 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.071857929229736 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.072215557098389 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.071670055389404 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.071950435638428 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.072112560272217 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.072005748748779 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.072428226470947 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.072158336639404 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.0718817710876465 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.072439670562744 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.071885108947754 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.072041988372803 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.072113513946533 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.072080612182617 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.072221279144287 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.0719709396362305 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.071987152099609 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.071938991546631 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.072324275970459 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.072481632232666 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.07216215133667 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.07211446762085 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.072135925292969 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.072214126586914 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.0722174644470215 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.072019100189209 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.071481704711914 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.072031021118164 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.071854114532471 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.072032451629639 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.071957111358643 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.07213830947876 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.072314739227295 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.072110176086426 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.072038650512695 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.071752548217773 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.071732521057129 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.071715831756592 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.07234001159668 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.072279930114746 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.072098731994629 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.072153568267822 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.0722737312316895 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.072350025177002 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.072229385375977 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.072163105010986 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.071720600128174 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.072180271148682 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.072205543518066 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.07193660736084 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.071766376495361 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.07168436050415 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.072051048278809 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.072170734405518 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.071825981140137 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.071458339691162 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.0717291831970215 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.0719990730285645 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.072004795074463 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.072323799133301 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.071820259094238 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.071976661682129 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.072022438049316 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.071806907653809 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.072051525115967 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.072263240814209 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.072188854217529 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.072322368621826 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.072056293487549 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.071523189544678 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.071737766265869 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.0719685554504395 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.07228946685791 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.071675777435303 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.072258949279785 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.072198390960693 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.071744441986084 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.071872711181641 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.0720133781433105 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.072289943695068 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.071564674377441 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.072144508361816 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.072000026702881 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.071991443634033 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.070985794067383 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.071650505065918 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.071960926055908 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.071681499481201 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.0716753005981445 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.071670055389404 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.071332931518555 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.071731090545654 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.071425914764404 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.07212495803833 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.071609973907471 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.071869850158691 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.071744441986084 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.070611953735352 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.071504592895508 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.072383403778076 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.072476387023926 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.072486877441406 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.071998119354248 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.072184085845947 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.0714311599731445 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.071132659912109 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.071215629577637 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.0724945068359375 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.071651458740234 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.07186222076416 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.071870803833008 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.072117328643799 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.072299480438232 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.071587085723877 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.071868419647217 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.071999549865723 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.0721917152404785 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.071938991546631 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.07192325592041 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.0718302726745605 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.071529865264893 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.071824550628662 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.071858882904053 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.071595191955566 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.071745872497559 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.071743488311768 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.071858882904053 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.071969985961914 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.072022914886475 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.071642875671387 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.071641445159912 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.071979522705078 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.072264671325684 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.071613311767578 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.0717973709106445 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.072142124176025 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.071639537811279 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.072229385375977 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.072088241577148 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.072026252746582 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.071281909942627 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.071850299835205 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.071506977081299 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.071193695068359 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.071662425994873 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.071972370147705 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.072477340698242 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.072139263153076 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.07186222076416 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.071470737457275 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.072062015533447 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.071839809417725 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.071764945983887 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.07210111618042 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.0714311599731445 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.071836471557617 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.071569919586182 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.071228504180908 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.071944713592529 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.072118759155273 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.071874141693115 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.07183837890625 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.071691989898682 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.071769714355469 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.071824550628662 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.072052955627441 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.072113513946533 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.071794033050537 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.071885108947754 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.071829319000244 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.071559429168701 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.0715155601501465 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.072086334228516 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.071983337402344 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.0710883140563965 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.072051048278809 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.071612358093262 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.071359634399414 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.07219934463501 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.072150707244873 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.0720086097717285 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.071839809417725 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.071737766265869 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.072150707244873 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.071822166442871 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.0720014572143555 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.071777820587158 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.0716142654418945 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.071971416473389 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.071354389190674 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.071789741516113 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.071631908416748 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.072622776031494 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.071819305419922 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.071974754333496 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.071524143218994 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.071683883666992 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.071634292602539 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.071292877197266 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.071727275848389 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.071808815002441 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.071990489959717 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.071859836578369 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.0714311599731445 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.071431636810303 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.072220325469971 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.071340084075928 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.072088241577148 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.072047233581543 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.07160758972168 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.071991443634033 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.071747779846191 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.071342468261719 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.071636199951172 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.071712493896484 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.071958541870117 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.0721755027771 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.072112560272217 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.072117328643799 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.072047233581543 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.070889472961426 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.072132587432861 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.071732997894287 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.071767807006836 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.071946620941162 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.071794033050537 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.071130752563477 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.071857929229736 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.0719146728515625 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.071966171264648 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.071922302246094 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.071802616119385 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.071660995483398 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.071328163146973 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.072165012359619 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.071990489959717 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.071959495544434 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.072178840637207 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.071922302246094 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.0716094970703125 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.072211742401123 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.071383476257324 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.072218418121338 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.071791172027588 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.072129249572754 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.072079658508301 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.072349548339844 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.072031021118164 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.071354866027832 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.0721354484558105 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.072150230407715 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.072077751159668 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.071957111358643 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.07133150100708 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.071323394775391 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.072243690490723 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.0718488693237305 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.0718841552734375 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.071937561035156 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.071965217590332 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.070943832397461 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.071503639221191 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.072061061859131 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.0720319747924805 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.072160243988037 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.07188081741333 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.071868896484375 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.071813583374023 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.071637153625488 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.072335720062256 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.072268962860107 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.072214126586914 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.071868896484375 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.072126388549805 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.071503639221191 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.070752143859863 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.0712714195251465 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.072276592254639 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.072066783905029 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.07241153717041 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.071907043457031 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.071372032165527 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.072004795074463 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.0722479820251465 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.072037220001221 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.072141647338867 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.07208776473999 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.072105884552002 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.071213722229004 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.071837902069092 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.0713958740234375 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.072261333465576 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.072062969207764 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.072154998779297 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.072827339172363 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.071674346923828 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.0718183517456055 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.071303367614746 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.071865558624268 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.072059154510498 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.071986198425293 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.072225093841553 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.071892738342285 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.072016716003418 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.071589946746826 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.072147369384766 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.0720601081848145 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.072129726409912 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.071691036224365 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.07204532623291 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.071529865264893 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.071096897125244 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.072108268737793 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.072089672088623 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.072310447692871 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.072042942047119 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.071735858917236 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.07206916809082 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.071718215942383 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.072073459625244 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.072256565093994 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.072014808654785 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.072428226470947 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.072024345397949 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.071965217590332 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.071364402770996 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.072007179260254 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.072085380554199 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.071994781494141 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.072152614593506 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.071547508239746 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.071630954742432 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.071028709411621 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.072081565856934 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.072007656097412 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.07187032699585 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.072196006774902 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.071939945220947 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.072301864624023 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.071630001068115 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.072208404541016 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.072196006774902 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.072108268737793 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.071934223175049 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.071590900421143 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.071706295013428 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.072218418121338 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.072117805480957 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.072052001953125 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.072006702423096 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.071928977966309 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.07138204574585 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.071865081787109 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.071224212646484 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.072023391723633 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.072035312652588 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.07199239730835 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.071824550628662 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.071910381317139 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.0719804763793945 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.071856498718262 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.072227954864502 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.072102069854736 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.072300910949707 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.071841716766357 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.072181224822998 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.072088718414307 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.071930885314941 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.072041034698486 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.07212495803833 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.072195053100586 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.072155952453613 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.071669101715088 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.07224702835083 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.071303367614746 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.072089195251465 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.072279930114746 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.072134971618652 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.072000026702881 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.07225227355957 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.071872711181641 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.0720086097717285 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.0721354484558105 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.07200288772583 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.0721893310546875 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.071752548217773 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.072065830230713 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.071808815002441 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.071451663970947 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.072325229644775 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.072193145751953 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.072154998779297 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.072358131408691 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.072129249572754 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.071931838989258 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.07183837890625 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.072255611419678 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.072154998779297 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.071786403656006 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.0719990730285645 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.071922779083252 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.071911334991455 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.072256565093994 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.07208251953125 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.072216510772705 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.0724358558654785 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.072129726409912 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.071840763092041 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.071928977966309 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.071605205535889 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.072144508361816 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.071817398071289 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.0716047286987305 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.071891784667969 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.071331977844238 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.072394847869873 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.071483612060547 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.072352409362793 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.0723700523376465 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.072429180145264 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.071900367736816 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.071656227111816 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.071951389312744 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.071504592895508 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.07191276550293 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.071987152099609 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.072578430175781 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.071934223175049 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.070959568023682 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.0719990730285645 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.0711517333984375 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.0719757080078125 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.0721516609191895 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.0696587562561035 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.071961879730225 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.0704665184021 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.074320316314697 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.071015357971191 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.072561740875244 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.072272777557373 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.071256637573242 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.071636199951172 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.070896148681641 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.071465969085693 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.072436809539795 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.071872234344482 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.071955680847168 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.0713300704956055 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.07148551940918 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.071605682373047 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.072075843811035 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.072150230407715 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.072246551513672 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.072999000549316 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.071727275848389 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.071702003479004 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.071086406707764 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.072196006774902 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.072512626647949 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.0758233070373535 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.074395656585693 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.071046352386475 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.071331024169922 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.070927143096924 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.073256015777588 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.071344375610352 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.072263240814209 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.072957515716553 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.070398330688477 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.071417808532715 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.071348190307617 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.071702003479004 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.072711944580078 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.07247257232666 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.072516918182373 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.071268081665039 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.071842670440674 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.071727275848389 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.072349548339844 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.072556018829346 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.072055339813232 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.071938514709473 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.071888446807861 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.0717363357543945 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.071451663970947 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.072386741638184 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.0719146728515625 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.072517395019531 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.072535037994385 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.071704864501953 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.072176456451416 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.070385456085205 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.070401668548584 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.0714545249938965 + } + ] + }, + { + "bit": 9, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.993472099304199 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.993544340133667 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.993838310241699 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.993795394897461 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.993529796600342 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.9934473037719727 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.993574380874634 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.9937496185302734 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.993617296218872 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.9935991764068604 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.9935450553894043 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.9933602809906006 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.99350643157959 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.9932849407196045 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.9934773445129395 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.9935829639434814 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.993466854095459 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.993579387664795 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.9935784339904785 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.993464469909668 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.993534564971924 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.9934258460998535 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.9936792850494385 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.9935591220855713 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.9935011863708496 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.9935555458068848 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.9934117794036865 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.9931840896606445 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.9935097694396973 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.9934558868408203 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.9934334754943848 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.993466854095459 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.993525981903076 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.993532419204712 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.9933550357818604 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.9937238693237305 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.9935147762298584 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.9934728145599365 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.9935131072998047 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.9936153888702393 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.993446111679077 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.993520736694336 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.993574380874634 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.993652582168579 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.9937613010406494 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.9932985305786133 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.9935410022735596 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.9932031631469727 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.9932308197021484 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.993412733078003 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.9934942722320557 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.9937570095062256 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.9936745166778564 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.993718385696411 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.9934489727020264 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.99336314201355 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.9935827255249023 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.9935860633850098 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.9931998252868652 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.9935107231140137 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.993593215942383 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.9936630725860596 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.9938430786132812 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.9936470985412598 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.993739366531372 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.993708610534668 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.9935269355773926 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.9933860301971436 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.993701219558716 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.9933888912200928 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.9936299324035645 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.9935460090637207 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.9936840534210205 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.9933364391326904 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.9935193061828613 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.9935269355773926 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.993762493133545 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.9935598373413086 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.9935190677642822 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.9935684204101562 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.993485927581787 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.9936110973358154 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.993407726287842 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.993725538253784 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.9937050342559814 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.993659257888794 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.993777275085449 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.9938125610351562 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.993762493133545 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.993630886077881 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.99345064163208 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.9933156967163086 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.9934632778167725 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.993622303009033 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.9937453269958496 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.9936299324035645 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.9937357902526855 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.993838310241699 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.9935526847839355 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.9934773445129395 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.9938364028930664 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.993607759475708 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.9934394359588623 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.993861436843872 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.9934985637664795 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.993635416030884 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.9935591220855713 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.993500232696533 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.9934945106506348 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.993372678756714 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.993485927581787 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.9935319423675537 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.99348783493042 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.9934334754943848 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.9937915802001953 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.993575096130371 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.9936776161193848 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.9934608936309814 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.993192672729492 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.9936282634735107 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.993577003479004 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.993946075439453 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.9937710762023926 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.9935474395751953 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.993788242340088 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.993246078491211 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.993483781814575 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.9940762519836426 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.992924451828003 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.9930524826049805 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.9936180114746094 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.9935860633850098 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.9935529232025146 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.993583917617798 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.993853807449341 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.9930098056793213 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.993460178375244 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.9934017658233643 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.9933643341064453 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.9932291507720947 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.993842601776123 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.993471145629883 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.993917226791382 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.9938271045684814 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.9937140941619873 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.9937350749969482 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.993553638458252 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.9936728477478027 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.9933459758758545 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.992853879928589 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.9939937591552734 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.9936728477478027 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.9934751987457275 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.993729829788208 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.9933483600616455 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.9934000968933105 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.9928462505340576 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.9929893016815186 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.993722677230835 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.9934160709381104 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.9933602809906006 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.993548631668091 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.9937407970428467 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.9936106204986572 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.9930386543273926 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.993363618850708 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.9938409328460693 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.9935121536254883 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.9936001300811768 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.993561267852783 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.9934098720550537 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.9940972328186035 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.993997812271118 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.993795394897461 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.993774175643921 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.9935903549194336 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.9936728477478027 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.9936647415161133 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.9934043884277344 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.9931223392486572 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.993577718734741 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.9934916496276855 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.993665933609009 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.9937632083892822 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.9937374591827393 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.993851661682129 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.993293285369873 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.9934709072113037 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.9932727813720703 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.993712902069092 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.9936118125915527 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.9923665523529053 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.9932823181152344 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.9934847354888916 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.993746280670166 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.9937119483947754 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.993354082107544 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.993539810180664 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.9931721687316895 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.9937238693237305 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.9938507080078125 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.993309736251831 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.993283987045288 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.9936516284942627 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.993892192840576 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.993643045425415 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.9938602447509766 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.9937171936035156 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.993396520614624 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.993464231491089 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.9935848712921143 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.993730068206787 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.9935948848724365 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.9935367107391357 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.993481397628784 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.9933602809906006 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.9933104515075684 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.9935948848724365 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.993481397628784 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.99357533454895 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.993595600128174 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.993340492248535 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.9937965869903564 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.99371600151062 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.9936017990112305 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.993741035461426 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.9932754039764404 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.9935436248779297 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.9933128356933594 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.993277072906494 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.9933879375457764 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.9938981533050537 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.993746280670166 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.9931437969207764 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.9932613372802734 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.9940872192382812 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.993640661239624 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.993964195251465 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.9935569763183594 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.9937000274658203 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.9943041801452637 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.993340492248535 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.9928946495056152 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.994206666946411 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.9928395748138428 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.9937169551849365 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.9936068058013916 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.99333119392395 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.99371600151062 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.993523359298706 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.9937641620635986 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.993739366531372 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.993727922439575 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.9938182830810547 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.9933717250823975 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.9937314987182617 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.9932608604431152 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.993868827819824 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.9931557178497314 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.993664264678955 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.993305206298828 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.993546962738037 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.9932286739349365 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.9935405254364014 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.9937965869903564 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.993290901184082 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.9935336112976074 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.9936647415161133 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.99350643157959 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.993382692337036 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.9933974742889404 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.9931483268737793 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.9937829971313477 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.9935615062713623 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.9935195446014404 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.993490695953369 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.9928624629974365 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.9935805797576904 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.9939863681793213 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.9930613040924072 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.9936118125915527 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.9934027194976807 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.993513584136963 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.9935555458068848 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.9928526878356934 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.993260383605957 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.9935965538024902 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.9935848712921143 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.99377179145813 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.9938061237335205 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.993285655975342 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.9933536052703857 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.9939136505126953 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.9932785034179688 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.9934825897216797 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.9937922954559326 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.993530750274658 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.9934804439544678 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.9931840896606445 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.9936981201171875 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.99336576461792 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.9940152168273926 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.9937245845794678 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.9938268661499023 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.993990659713745 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.993399143218994 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.99311900138855 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.993769645690918 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.9937734603881836 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.9937922954559326 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.993812084197998 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.993595838546753 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.992828130722046 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.9930148124694824 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.993117094039917 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.993628978729248 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.993838310241699 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.9936530590057373 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.9938535690307617 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.9936904907226562 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.993145227432251 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.9927008152008057 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.9935407638549805 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.993720293045044 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.993603229522705 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.993790626525879 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.9938313961029053 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.993497610092163 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.9928603172302246 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.993790626525879 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.9938278198242188 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.993729829788208 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.993614673614502 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.993173837661743 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.9937257766723633 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.993156671524048 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.99397349357605 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.993690013885498 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.9938957691192627 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.9938442707061768 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.993698835372925 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.9940741062164307 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.993828296661377 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.993906021118164 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.99371600151062 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.9934985637664795 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.9937727451324463 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.9936299324035645 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.9935879707336426 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.992978096008301 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.993785858154297 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.993712902069092 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.993588924407959 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.993701219558716 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.9938137531280518 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.993333578109741 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.993682622909546 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.9935879707336426 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.993734359741211 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.9935641288757324 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.993121385574341 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.993460178375244 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.993349075317383 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.993816375732422 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.9936716556549072 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.9940743446350098 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.9937350749969482 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.9936299324035645 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.9931986331939697 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.993698835372925 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.993375539779663 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.9937539100646973 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.9938740730285645 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.9938502311706543 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.9938066005706787 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.993037462234497 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.9935495853424072 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.99356746673584 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.9938888549804688 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.993593215942383 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.993605136871338 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.993446111679077 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.9935975074768066 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.993589162826538 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.9934394359588623 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.9937615394592285 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.993781328201294 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.9937667846679688 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.9932329654693604 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.993183135986328 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.9930880069732666 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.993501901626587 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.993720293045044 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.993736505508423 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.99387526512146 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.993753671646118 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.993694543838501 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.9930834770202637 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.993867874145508 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.9937829971313477 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.9937691688537598 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.993943929672241 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.993546962738037 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.9936234951019287 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.993129253387451 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.9934868812561035 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.9936256408691406 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.9935505390167236 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.993664264678955 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.993593454360962 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.993675470352173 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.9933464527130127 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.9934334754943848 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.993724822998047 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.9937453269958496 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.993760108947754 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.993497610092163 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.9938125610351562 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.993489980697632 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.9937193393707275 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.9936482906341553 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.993626594543457 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.9936697483062744 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.9937033653259277 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.9937872886657715 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.9931437969207764 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.9932861328125 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.9936954975128174 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.9937829971313477 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.993548631668091 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.993638515472412 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.9930098056793213 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.993377447128296 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.9932823181152344 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.993720293045044 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.99356746673584 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.993601083755493 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.993673324584961 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.9931869506835938 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.9931018352508545 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.9938406944274902 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.9936716556549072 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.9936869144439697 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.993696928024292 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.9936764240264893 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.9927785396575928 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.993293523788452 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.993325710296631 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.993666887283325 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.993662118911743 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.9934654235839844 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.9936249256134033 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.9933927059173584 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.9935925006866455 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.9940743446350098 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.9936368465423584 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.9938182830810547 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.993717908859253 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.9936163425445557 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.9930996894836426 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.9934566020965576 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.9935431480407715 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.9936437606811523 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.993624448776245 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.993919610977173 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.993605136871338 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.993011713027954 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.992917537689209 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.9936087131500244 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.993831157684326 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.9937193393707275 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.9938018321990967 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.993605136871338 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.9936256408691406 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.993138551712036 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.9932126998901367 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.9937527179718018 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.993739366531372 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.9937915802001953 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.9933464527130127 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.9933910369873047 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.993591547012329 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.993497610092163 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.9938137531280518 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.9937562942504883 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.993242025375366 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.9937667846679688 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.9934792518615723 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.9930784702301025 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.9936587810516357 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.99381160736084 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.9937140941619873 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.9932754039764404 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.9933431148529053 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.9933698177337646 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.9935898780822754 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.993469715118408 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.993504524230957 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.993452310562134 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.991504192352295 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.993450403213501 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.992504119873047 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.9940590858459473 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.9933385848999023 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.9930546283721924 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.9933362007141113 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.992540121078491 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.9933855533599854 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.993004560470581 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.9940237998962402 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.99369215965271 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.993295669555664 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.994514226913452 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.992061138153076 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.993077039718628 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.9930524826049805 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.994086265563965 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.9935367107391357 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.9935364723205566 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.993706703186035 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.991920232772827 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.993227481842041 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.991987466812134 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.9945926666259766 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.991899013519287 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.996307611465454 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.9961764812469482 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.991955280303955 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.993867874145508 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.9922804832458496 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.9934659004211426 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.9942753314971924 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.9938297271728516 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.99552321434021 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.992893695831299 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.9928414821624756 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.9939894676208496 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.994246006011963 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.99391508102417 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.9935061931610107 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.99375057220459 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.9929375648498535 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.9935836791992188 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.9938766956329346 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.9941415786743164 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.9931323528289795 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.9935708045959473 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.992992401123047 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.9929158687591553 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.9934678077697754 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.9927029609680176 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.9946296215057373 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.9942548274993896 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.9937515258789062 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.994018316268921 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.993832588195801 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.9936106204986572 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.9926974773406982 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.9929850101470947 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.9897754192352295 + } + ] + }, + { + "bit": 10, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.9224393367767334 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.9222536087036133 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.922328472137451 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.9222652912139893 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.9223954677581787 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.9224836826324463 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.922121047973633 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.9225730895996094 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.9220497608184814 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.9224486351013184 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.9221060276031494 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.922427177429199 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.9224510192871094 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.922271966934204 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.9224205017089844 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.9224853515625 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.9225234985351562 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.9223828315734863 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.9221582412719727 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.9222445487976074 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.922626256942749 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.92244029045105 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.922531843185425 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.922034502029419 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.9220941066741943 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.9221689701080322 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.921846628189087 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.9221997261047363 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.922428846359253 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.922685384750366 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.9224140644073486 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.922144651412964 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.922420024871826 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.9219744205474854 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.9219133853912354 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.922539472579956 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.9224560260772705 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.922297239303589 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.9224560260772705 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.922110080718994 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.9225997924804688 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.9222428798675537 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.922480344772339 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.922447681427002 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.9223926067352295 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.922431707382202 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.9223239421844482 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.9224889278411865 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.9225881099700928 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.9224648475646973 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.922579526901245 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.922224998474121 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.9229257106781006 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.9224963188171387 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.922297954559326 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.922422409057617 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.9223148822784424 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.922281503677368 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.9225142002105713 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.9220407009124756 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.922211170196533 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.9221465587615967 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.9217822551727295 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.92242431640625 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.922091484069824 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.92230486869812 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.9222278594970703 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.9223792552948 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.9220354557037354 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.9224720001220703 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.9223482608795166 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.922327995300293 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.922231912612915 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.922661304473877 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.922273635864258 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.9225587844848633 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.921893835067749 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.9225339889526367 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.9223058223724365 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.922238349914551 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.922013759613037 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.9223568439483643 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.922355890274048 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.9222066402435303 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.9222629070281982 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.9229564666748047 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.9220969676971436 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.9222075939178467 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.922367572784424 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.9223649501800537 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.9224581718444824 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.9223475456237793 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.92238712310791 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.922220468521118 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.9221436977386475 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.92242431640625 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.9222166538238525 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.9222323894500732 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.922701120376587 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.922614574432373 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.922166585922241 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.9221949577331543 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.92252254486084 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.9220802783966064 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.922558307647705 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.9223439693450928 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.922255277633667 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.9225170612335205 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.922481060028076 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.9229490756988525 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.922455072402954 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.922555685043335 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.9222424030303955 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.922316551208496 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.9223501682281494 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.9227190017700195 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.9227795600891113 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.922334671020508 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.922516107559204 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.9221363067626953 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.922178268432617 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.92193341255188 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.9225542545318604 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.922344923019409 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.922407865524292 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.9225375652313232 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.9224836826324463 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.922518014907837 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.9224870204925537 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.922546863555908 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.922718048095703 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.9221503734588623 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.9224753379821777 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.922332286834717 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.922427177429199 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.921995162963867 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.922248601913452 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.922583818435669 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.922581911087036 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.9226441383361816 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.9225656986236572 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.9223852157592773 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.9222259521484375 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.921931505203247 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.9222447872161865 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.9224603176116943 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.9221041202545166 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.922553300857544 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.9222373962402344 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.9224495887756348 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.9215359687805176 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.9220681190490723 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.9223294258117676 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.922499179840088 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.922135829925537 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.9224283695220947 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.9227824211120605 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.922189712524414 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.922485828399658 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.9224531650543213 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.922691822052002 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.9225597381591797 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.922443151473999 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.9214625358581543 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.9220263957977295 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.9227678775787354 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.9226558208465576 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.923008680343628 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.922437906265259 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.922733783721924 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.9218811988830566 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.9218361377716064 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.922229290008545 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.9223196506500244 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.922292470932007 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.922435760498047 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.9223718643188477 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.922459363937378 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.92263126373291 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.9221906661987305 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.92179799079895 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.9225471019744873 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.922584056854248 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.922391653060913 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.9223599433898926 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.9223294258117676 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.922250270843506 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.922746181488037 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.922556161880493 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.922285318374634 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.9222288131713867 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.9232704639434814 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.9223663806915283 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.922447681427002 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.92197585105896 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.9220550060272217 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.922722339630127 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.9223592281341553 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.9224843978881836 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.9223239421844482 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.922372579574585 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.9224393367767334 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.9223384857177734 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.922541379928589 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.922302722930908 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.922412157058716 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.9223780632019043 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.922231912612915 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.921983003616333 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.9220356941223145 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.922314167022705 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.9226863384246826 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.9223556518554688 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.922678232192993 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.9223644733428955 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.9221949577331543 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.9223501682281494 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.9226412773132324 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.9222805500030518 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.922114849090576 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.922043800354004 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.922473907470703 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.922152042388916 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.922268867492676 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.9224514961242676 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.9225223064422607 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.922541379928589 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.9220454692840576 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.9223427772521973 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.9222233295440674 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.9221129417419434 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.9225881099700928 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.922525405883789 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.9227657318115234 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.922295331954956 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.922274351119995 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.922689914703369 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.922407865524292 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.9226534366607666 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.922605276107788 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.921875238418579 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.9222323894500732 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.9223999977111816 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.9218413829803467 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.922637701034546 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.9226813316345215 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.922250270843506 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.9223897457122803 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.922339677810669 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.9224393367767334 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.922802686691284 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.922415018081665 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.9222774505615234 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.9223649501800537 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.922412157058716 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.9220316410064697 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.922543525695801 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.92250657081604 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.9230294227600098 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.9224812984466553 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.92250657081604 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.9218497276306152 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.922621726989746 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.922525405883789 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.9218223094940186 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.9229047298431396 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.922433853149414 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.9225404262542725 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.9224531650543213 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.9226670265197754 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.922528028488159 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.922388792037964 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.9217960834503174 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.922576665878296 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.922382116317749 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.922257661819458 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.9228837490081787 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.922511100769043 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.921903610229492 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.9222471714019775 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.9222753047943115 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.922598361968994 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.9223945140838623 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.922412157058716 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.922553300857544 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.9223833084106445 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.921875238418579 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.9225573539733887 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.9223718643188477 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.9221980571746826 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.922642230987549 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.9219353199005127 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.9219090938568115 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.9224677085876465 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.9224908351898193 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.9224188327789307 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.9224393367767334 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.9224674701690674 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.922539472579956 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.9220492839813232 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.9220073223114014 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.9224674701690674 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.9224655628204346 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.9225361347198486 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.9219324588775635 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.92256236076355 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.9224205017089844 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.922264814376831 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.9225313663482666 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.9223954677581787 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.922269105911255 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.9225881099700928 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.922877788543701 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.9224257469177246 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.9224252700805664 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.9223392009735107 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.922452926635742 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.9224328994750977 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.922475814819336 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.92193341255188 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.922513723373413 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.9229302406311035 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.9223990440368652 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.9224190711975098 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.922389268875122 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.9221436977386475 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.921846628189087 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.922321081161499 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.922664165496826 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.9224560260772705 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.9224424362182617 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.9223766326904297 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.922606945037842 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.922398090362549 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.922293186187744 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.9220950603485107 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.922489881515503 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.922541379928589 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.9225215911865234 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.922788619995117 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.922163486480713 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.921574354171753 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.9218571186065674 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.922637701034546 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.9225261211395264 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.9224116802215576 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.9226272106170654 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.9221932888031006 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.922142744064331 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.9224953651428223 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.922595500946045 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.9225287437438965 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.9226057529449463 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.922396183013916 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.9220988750457764 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.922201633453369 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.922048807144165 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.9225587844848633 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.9225523471832275 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.9224812984466553 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.9230127334594727 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.9221792221069336 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.921999454498291 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.9215519428253174 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.922513723373413 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.92248797416687 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.922574996948242 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.922332525253296 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.9223392009735107 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.922431707382202 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.9218568801879883 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.9225871562957764 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.922455072402954 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.922536611557007 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.92207670211792 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.9228317737579346 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.9225988388061523 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.921647310256958 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.922539472579956 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.922427177429199 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.922653913497925 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.9225540161132812 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.9223756790161133 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.9224793910980225 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.922534227371216 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.922556161880493 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.9225006103515625 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.9224019050598145 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.922844648361206 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.9225258827209473 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.922478437423706 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.9220850467681885 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.922553300857544 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.9224889278411865 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.9225423336029053 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.922450542449951 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.9219415187835693 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.9224612712860107 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.9220027923583984 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.9225425720214844 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.922478437423706 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.922290086746216 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.922598361968994 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.922374725341797 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.922194242477417 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.921994209289551 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.92246150970459 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.9224696159362793 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.9224822521209717 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.922581672668457 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.921908140182495 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.922314405441284 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.922253131866455 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.92248797416687 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.9225258827209473 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.922417163848877 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.9226577281951904 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.9220528602600098 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.9220261573791504 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.922085762023926 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.9225828647613525 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.9225332736968994 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.9222805500030518 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.9223792552948 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.9220755100250244 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.9220597743988037 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.9221138954162598 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.9225974082946777 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.922297954559326 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.9225850105285645 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.922473907470703 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.9223504066467285 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.922269105911255 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.9223062992095947 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.9223811626434326 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.922375202178955 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.922670364379883 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.922471046447754 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.922133684158325 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.9225142002105713 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.922147512435913 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.922532320022583 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.9225714206695557 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.922337532043457 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.9225666522979736 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.9226019382476807 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.921828269958496 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.922426223754883 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.9224393367767334 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.9224853515625 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.922464609146118 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.9223337173461914 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.9222195148468018 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.9222166538238525 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.921736001968384 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.9224627017974854 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.922468423843384 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.9224283695220947 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.922344923019409 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.9230220317840576 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.9217875003814697 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.922013759613037 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.9223241806030273 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.922501802444458 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.9224064350128174 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.922426223754883 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.922273635864258 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.9227135181427 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.922402858734131 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.9223601818084717 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.9225656986236572 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.922553300857544 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.9225776195526123 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.9218108654022217 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.9221606254577637 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.9224929809570312 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.922290563583374 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.9223196506500244 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.9222476482391357 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.922316789627075 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.9219725131988525 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.9222233295440674 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.921982526779175 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.9224655628204346 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.9223766326904297 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.9226226806640625 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.9222943782806396 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.922335624694824 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.9220850467681885 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.922043800354004 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.9223291873931885 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.922427177429199 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.9223062992095947 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.9223270416259766 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.9221994876861572 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.9216575622558594 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.921517848968506 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.922421455383301 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.9224655628204346 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.919445037841797 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.922428607940674 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.9211583137512207 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.923677682876587 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.9217660427093506 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.9223546981811523 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.922508716583252 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.921661138534546 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.9222476482391357 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.9216501712799072 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.9224929809570312 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.9220821857452393 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.922210216522217 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.9222636222839355 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.9212470054626465 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.921898365020752 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.9221951961517334 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.9223692417144775 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.9222493171691895 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.922422409057617 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.923081636428833 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.922276496887207 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.922435998916626 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.921320915222168 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.9219541549682617 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.9225270748138428 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.921889066696167 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.9247021675109863 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.9199156761169434 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.921983003616333 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.921975612640381 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.9234185218811035 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.9217591285705566 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.9227025508880615 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.92291259765625 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.9218342304229736 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.9219343662261963 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.922067165374756 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.922553300857544 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.9237544536590576 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.9222795963287354 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.9228129386901855 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.9216487407684326 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.922180414199829 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.9220211505889893 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.922816276550293 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.923196792602539 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.922327995300293 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.922323226928711 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.9222512245178223 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.9221465587615967 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.922020196914673 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.9222662448883057 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.921922445297241 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.9227395057678223 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.9225785732269287 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.922452449798584 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.9225587844848633 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.9214210510253906 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.9212677478790283 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.919799327850342 + } + ] + }, + { + "bit": 11, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.8571712970733643 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.857137441635132 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.857120990753174 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.8571465015411377 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.857295036315918 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.857117176055908 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.8569037914276123 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.856928825378418 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.8569860458374023 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.8570010662078857 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.8567492961883545 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.8569717407226562 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.8570637702941895 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.8571152687072754 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.857229232788086 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.857149362564087 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.8571455478668213 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.856954336166382 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.856919765472412 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.8570661544799805 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.857032537460327 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.8571667671203613 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.85715651512146 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.8572561740875244 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.8571858406066895 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.85721492767334 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.8571677207946777 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.8568568229675293 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.85713267326355 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.8570950031280518 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.857044219970703 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.8568410873413086 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.8568551540374756 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.8572781085968018 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.8570785522460938 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.8570351600646973 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.8572580814361572 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.857292652130127 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.856879949569702 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.8570051193237305 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.8571438789367676 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.8569729328155518 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.857241630554199 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.8571200370788574 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.8571152687072754 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.8570609092712402 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.857236623764038 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.8569321632385254 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.856807231903076 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.85707950592041 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.8572850227355957 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.8571078777313232 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.856992244720459 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.857407808303833 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.8568716049194336 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.8571512699127197 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.8571836948394775 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.8568294048309326 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.856865882873535 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.8570690155029297 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.8571465015411377 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.8573153018951416 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.857517957687378 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.856949806213379 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.857051372528076 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.8569893836975098 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.8571417331695557 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.85711669921875 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.857128143310547 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.857067108154297 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.8570408821105957 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.8572421073913574 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.8571221828460693 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.857034921646118 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.857030153274536 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.8571555614471436 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.8572003841400146 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.8572094440460205 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.857036828994751 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.857177734375 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.857102394104004 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.8572936058044434 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.857117176055908 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.8571126461029053 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.8572933673858643 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.8570570945739746 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.8575539588928223 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.8571410179138184 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.857106924057007 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.857283353805542 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.857114791870117 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.857247829437256 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.857041358947754 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.8571417331695557 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.8571598529815674 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.856982946395874 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.857203245162964 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.857404947280884 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.8570210933685303 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.857142925262451 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.857255458831787 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.8573195934295654 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.856907367706299 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.857478618621826 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.857069492340088 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.8570170402526855 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.8573074340820312 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.8571999073028564 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.857001781463623 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.8567850589752197 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.857236623764038 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.857123374938965 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.8570284843444824 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.8570775985717773 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.8571271896362305 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.8568999767303467 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.8569796085357666 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.8571271896362305 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.8569157123565674 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.857325553894043 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.856853485107422 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.857067346572876 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.8568742275238037 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.857210874557495 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.856996536254883 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.8574864864349365 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.8571696281433105 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.8573038578033447 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.8569533824920654 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.856755256652832 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.857217788696289 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.857062816619873 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.8570632934570312 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.8570284843444824 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.8571741580963135 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.8568596839904785 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.857184410095215 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.8570151329040527 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.856997489929199 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.8570919036865234 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.8571224212646484 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.8570756912231445 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.85728120803833 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.857389211654663 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.8571722507476807 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.8570921421051025 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.8569812774658203 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.857135534286499 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.8571174144744873 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.8562281131744385 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.8576769828796387 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.8569469451904297 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.857069253921509 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.8570878505706787 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.8569395542144775 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.8570058345794678 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.856072425842285 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.8568551540374756 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.8569231033325195 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.8574044704437256 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.8566527366638184 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.85689377784729 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.8569884300231934 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.8565938472747803 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.8570785522460938 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.8567776679992676 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.8574891090393066 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.857285261154175 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.8570590019226074 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.857124090194702 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.8572540283203125 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.8572707176208496 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.857182502746582 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.856990337371826 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.8576595783233643 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.8570597171783447 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.8572237491607666 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.8570845127105713 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.8569717407226562 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.8574295043945312 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.857489585876465 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.8569095134735107 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.85722279548645 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.8571345806121826 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.8572874069213867 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.856804370880127 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.857228994369507 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.8569064140319824 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.857400417327881 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.85697078704834 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.857006788253784 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.8553519248962402 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.856956720352173 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.856964111328125 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.8570094108581543 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.857370138168335 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.8572025299072266 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.857227087020874 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.8569281101226807 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.85722279548645 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.8573920726776123 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.8568882942199707 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.856905460357666 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.8570637702941895 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.857180595397949 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.8571014404296875 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.8568625450134277 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.857027769088745 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.856983184814453 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.8572099208831787 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.8570399284362793 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.8571317195892334 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.8575820922851562 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.857243061065674 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.8571629524230957 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.857268810272217 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.856905460357666 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.8571133613586426 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.856975793838501 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.8569281101226807 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.856987953186035 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.8572936058044434 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.8573455810546875 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.857243061065674 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.857091188430786 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.857170343399048 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.8567869663238525 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.8571696281433105 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.856807231903076 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.8570663928985596 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.857217788696289 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.857147455215454 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.8571629524230957 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.8567965030670166 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.8571317195892334 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.857170343399048 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.856910228729248 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.857184410095215 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.857292652130127 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.857161283493042 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.8577911853790283 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.857508897781372 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.8568127155303955 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.8571503162384033 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.8569424152374268 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.8572564125061035 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.857128143310547 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.8567116260528564 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.857218027114868 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.8572824001312256 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.856748342514038 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.857070207595825 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.8571059703826904 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.8572330474853516 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.8569529056549072 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.8571712970733643 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.85720157623291 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.8571059703826904 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.857105255126953 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.8569936752319336 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.8570775985717773 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.8573966026306152 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.8569676876068115 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.8570473194122314 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.8573975563049316 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.8565011024475098 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.8573105335235596 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.8572335243225098 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.857182741165161 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.8570830821990967 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.8569557666778564 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.8570399284362793 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.8570380210876465 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.857081651687622 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.8573570251464844 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.857043504714966 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.856987476348877 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.8568758964538574 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.857301950454712 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.856691360473633 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.8571925163269043 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.856934070587158 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.8573215007781982 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.8572356700897217 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.856485366821289 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.85693359375 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.8572301864624023 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.8569416999816895 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.857062816619873 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.857173204421997 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.8569931983947754 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.8573262691497803 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.8572146892547607 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.8567607402801514 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.8573155403137207 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.8570191860198975 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.8573083877563477 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.8572468757629395 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.856780529022217 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.857165813446045 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.8570008277893066 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.857346296310425 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.8570332527160645 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.857247829437256 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.857388973236084 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.856998920440674 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.8563592433929443 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.8571202754974365 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.8573031425476074 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.8571534156799316 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.8573617935180664 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.856990337371826 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.856415271759033 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.857086181640625 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.856661319732666 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.8571455478668213 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.8571951389312744 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.857093095779419 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.8573508262634277 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.8572685718536377 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.8571696281433105 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.8566601276397705 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.8572914600372314 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.8572146892547607 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.857133626937866 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.857280969619751 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.857142925262451 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.8571853637695312 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.8571298122406006 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.857346534729004 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.857257127761841 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.8574233055114746 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.857064723968506 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.8568062782287598 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.8569018840789795 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.856926202774048 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.8573145866394043 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.8572394847869873 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.8572511672973633 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.8570523262023926 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.856884717941284 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.857534885406494 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.857107639312744 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.857248067855835 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.857200860977173 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.85736083984375 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.8572776317596436 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.857469320297241 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.8573851585388184 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.8569772243499756 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.8571853637695312 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.857130765914917 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.8570992946624756 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.8572051525115967 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.857370138168335 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.8570973873138428 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.8566269874572754 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.8572330474853516 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.857170343399048 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.8571126461029053 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.856786012649536 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.856841802597046 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.8573195934295654 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.8573954105377197 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.8573031425476074 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.8573434352874756 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.8570170402526855 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.8572959899902344 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.8569416999816895 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.8568825721740723 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.857059955596924 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.8573215007781982 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.857152223587036 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.8571979999542236 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.857342481613159 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.856804847717285 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.8568780422210693 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.8571555614471436 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.8572206497192383 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.8572118282318115 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.85713267326355 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.8569772243499756 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.8569204807281494 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.857140064239502 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.856696128845215 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.857206344604492 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.857297420501709 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.857304811477661 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.8570494651794434 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.85654878616333 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.8569204807281494 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.8566126823425293 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.8573267459869385 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.857128143310547 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.857135057449341 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.8571293354034424 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.8571202754974365 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.8573520183563232 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.8573415279388428 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.857318639755249 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.857205390930176 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.8572487831115723 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.8571360111236572 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.8567140102386475 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.8567919731140137 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.857745409011841 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.8572397232055664 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.8573029041290283 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.857268810272217 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.857102870941162 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.857367515563965 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.856992483139038 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.8574626445770264 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.8572850227355957 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.857252836227417 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.8572158813476562 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.8570170402526855 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.8572301864624023 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.856980085372925 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.8570969104766846 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.857250213623047 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.857128143310547 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.8574023246765137 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.857325792312622 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.857218027114868 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.8574297428131104 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.8572893142700195 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.857133626937866 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.857144832611084 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.85703706741333 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.8571410179138184 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.8570077419281006 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.8570618629455566 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.856997489929199 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.857224225997925 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.8573296070098877 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.857163906097412 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.857025623321533 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.8571317195892334 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.856635332107544 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.856844902038574 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.8572580814361572 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.8571877479553223 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.857200860977173 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.857086181640625 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.8564839363098145 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.8573811054229736 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.856929063796997 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.8572263717651367 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.857203245162964 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.8572137355804443 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.857292413711548 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.8571276664733887 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.856884479522705 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.8575944900512695 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.857365369796753 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.85736346244812 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.8571243286132812 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.8572146892547607 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.856677532196045 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.85733962059021 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.856865882873535 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.857309341430664 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.8572659492492676 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.857245922088623 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.8571224212646484 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.856837511062622 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.8565521240234375 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.8572075366973877 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.8573882579803467 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.8571243286132812 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.8572330474853516 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.8571243286132812 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.8571043014526367 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.8568572998046875 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.856606960296631 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.857285261154175 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.8573718070983887 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.85722279548645 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.85725736618042 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.857415199279785 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.8571224212646484 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.8569395542144775 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.857409715652466 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.8571958541870117 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.8570878505706787 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.85715651512146 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.8569037914276123 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.8568990230560303 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.8565523624420166 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.8572208881378174 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.857355833053589 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.8569788932800293 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.8569414615631104 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.8570804595947266 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.85756778717041 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.857361316680908 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.8570311069488525 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.856998920440674 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.8504273891448975 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.857163906097412 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.856342077255249 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.8579812049865723 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.8572895526885986 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.8571271896362305 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.8571033477783203 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.8552207946777344 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.857137441635132 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.8568503856658936 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.8572940826416016 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.85752534866333 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.856877326965332 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.8574559688568115 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.8557939529418945 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.856908082962036 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.856407642364502 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.8576340675354004 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.857149362564087 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.8571770191192627 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.857344388961792 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.856088638305664 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.8568854331970215 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.856402635574341 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.8577029705047607 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.856475591659546 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.859501838684082 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.8581912517547607 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.8516132831573486 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.857384204864502 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.8560237884521484 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.8572065830230713 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.857609272003174 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.857478618621826 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.858347177505493 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.8564772605895996 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.8569092750549316 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.8571484088897705 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.8575451374053955 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.8574044704437256 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.857218027114868 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.857358455657959 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.8559470176696777 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.8572263717651367 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.8569839000701904 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.8574795722961426 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.857084035873413 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.8573131561279297 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.856997013092041 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.856743812561035 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.8569414615631104 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.8570806980133057 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.858095645904541 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.8576226234436035 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.8572449684143066 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.857320547103882 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.857126235961914 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.857111692428589 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.856806755065918 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.856931209564209 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.8573946952819824 + } + ] + }, + { + "bit": 12, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.8082284927368164 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.807908058166504 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.8079144954681396 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.8082122802734375 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.8082997798919678 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.8080267906188965 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.808323383331299 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.8079662322998047 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.807912588119507 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.8083066940307617 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.808237075805664 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.8079919815063477 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.8080341815948486 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.807891607284546 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.8079283237457275 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.8079631328582764 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.808234930038452 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.8079681396484375 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.8082799911499023 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.8079843521118164 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.808042287826538 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.8081436157226562 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.808164119720459 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.8080155849456787 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.8080739974975586 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.8078277111053467 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.8079917430877686 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.808076858520508 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.8082525730133057 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.808156967163086 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.8082962036132812 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.808007001876831 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.808229446411133 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.8078079223632812 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.80804705619812 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.8081603050231934 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.8080155849456787 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.80804705619812 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.8080894947052 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.807978391647339 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.8078908920288086 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.807912588119507 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.8082239627838135 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.8080079555511475 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.807959794998169 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.8080451488494873 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.807976484298706 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.808147668838501 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.8081753253936768 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.8082022666931152 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.80812406539917 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.8079967498779297 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.8084006309509277 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.80812406539917 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.8078627586364746 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.808008909225464 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.808107376098633 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.8081088066101074 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.8081159591674805 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.8080942630767822 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.8080976009368896 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.808230400085449 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.8080379962921143 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.808087110519409 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.808396816253662 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.8083035945892334 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.807955741882324 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.8081936836242676 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.8082022666931152 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.808032512664795 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.8079516887664795 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.8080196380615234 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.8079652786254883 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.8082549571990967 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.808140516281128 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.8080172538757324 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.80818772315979 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.8080193996429443 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.8082430362701416 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.808002471923828 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.8081114292144775 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.8079724311828613 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.8079333305358887 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.8082077503204346 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.8080668449401855 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.808162212371826 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.8080267906188965 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.808088779449463 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.808138370513916 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.8082027435302734 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.808422803878784 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.807997941970825 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.8081541061401367 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.8077685832977295 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.8078880310058594 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.808541774749756 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.80804705619812 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.807952642440796 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.808058738708496 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.8079941272735596 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.8076956272125244 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.807703733444214 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.8082003593444824 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.8080244064331055 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.808384656906128 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.808021306991577 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.8081088066101074 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.8080029487609863 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.807859182357788 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.808401107788086 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.8079833984375 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.808105945587158 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.8082714080810547 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.8078677654266357 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.8081777095794678 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.8082594871520996 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.807903528213501 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.8079617023468018 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.808436632156372 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.8080551624298096 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.8081424236297607 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.808048963546753 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.8079495429992676 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.8080716133117676 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.808101177215576 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.8079307079315186 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.808047294616699 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.8078267574310303 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.807996988296509 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.807976245880127 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.808335304260254 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.8080615997314453 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.807774305343628 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.8079495429992676 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.8082566261291504 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.8077774047851562 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.807659387588501 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.8080389499664307 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.807978868484497 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.8078908920288086 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.808149576187134 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.8079464435577393 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.8076531887054443 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.8078720569610596 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.8082869052886963 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.8078224658966064 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.807811975479126 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.808123826980591 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.807964324951172 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.8081207275390625 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.8073315620422363 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.8080031871795654 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.8081612586975098 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.8077046871185303 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.80818772315979 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.8081367015838623 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.8086421489715576 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.807662010192871 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.808131456375122 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.8078274726867676 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.8082756996154785 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.8080570697784424 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.808192729949951 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.8072707653045654 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.807694673538208 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.8084001541137695 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.80802059173584 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.808553695678711 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.808166265487671 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.808155059814453 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.8076393604278564 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.8077056407928467 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.807954788208008 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.8082172870635986 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.8080060482025146 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.808056354522705 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.8080129623413086 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.8078954219818115 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.808002471923828 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.807832956314087 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.807806968688965 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.8081095218658447 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.8082780838012695 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.808126926422119 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.807941198348999 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.8077659606933594 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.807838201522827 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.808147430419922 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.8078227043151855 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.8080625534057617 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.8080461025238037 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.809433698654175 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.8077664375305176 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.8081068992614746 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.808145761489868 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.807947874069214 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.8081443309783936 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.807974338531494 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.8081068992614746 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.8080813884735107 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.8078227043151855 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.807950735092163 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.807921886444092 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.8081023693084717 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.8082072734832764 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.807889938354492 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.8078010082244873 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.80790638923645 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.8077003955841064 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.807810068130493 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.8078908920288086 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.8082022666931152 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.807573080062866 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.8079233169555664 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.807969093322754 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.807845115661621 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.808070659637451 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.8080389499664307 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.807976245880127 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.8079121112823486 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.8076465129852295 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.807612180709839 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.807833671569824 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.8079700469970703 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.808042287826538 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.8079674243927 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.807633876800537 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.80774188041687 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.8080217838287354 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.8079535961151123 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.8076300621032715 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.8080837726593018 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.808206796646118 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.8079752922058105 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.8079240322113037 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.808037042617798 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.808013916015625 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.8076350688934326 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.807988405227661 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.8081209659576416 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.807356119155884 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.8080177307128906 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.80804705619812 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.807680130004883 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.8078975677490234 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.8081130981445312 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.807887315750122 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.8079633712768555 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.8081703186035156 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.8077378273010254 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.808302164077759 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.80763578414917 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.8080389499664307 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.8080921173095703 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.8076906204223633 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.807746648788452 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.8078155517578125 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.808002471923828 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.808173179626465 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.808087110519409 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.808227300643921 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.8075265884399414 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.807652473449707 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.808060646057129 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.8080408573150635 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.808016300201416 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.8079936504364014 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.808171272277832 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.808072566986084 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.8075814247131348 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.8082234859466553 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.8081679344177246 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.807783603668213 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.808176040649414 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.808130979537964 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.8078835010528564 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.808018684387207 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.8080556392669678 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.807683229446411 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.807701826095581 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.8080687522888184 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.8083510398864746 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.808089256286621 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.8081531524658203 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.8081486225128174 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.8079681396484375 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.807511806488037 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.8081140518188477 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.807967185974121 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.8079538345336914 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.8076348304748535 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.8076236248016357 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.8075203895568848 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.807986259460449 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.808229923248291 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.8080008029937744 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.8077971935272217 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.807739734649658 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.807800054550171 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.807579517364502 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.8079171180725098 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.808126926422119 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.8080756664276123 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.8081367015838623 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.807952880859375 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.807990074157715 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.808396577835083 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.807588577270508 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.808164358139038 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.8080360889434814 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.8079066276550293 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.8081541061401367 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.8083078861236572 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.8079545497894287 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.8076000213623047 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.80808424949646 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.8080735206604004 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.8078627586364746 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.807966470718384 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.8075625896453857 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.807746648788452 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.8081929683685303 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.808122158050537 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.8081796169281006 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.808159589767456 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.807690143585205 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.8078434467315674 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.8078227043151855 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.8074378967285156 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.808133363723755 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.808140516281128 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.8078973293304443 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.808276653289795 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.8075921535491943 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.8078761100769043 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.8076300621032715 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.8080501556396484 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.8079683780670166 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.807844400405884 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.80808424949646 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.807748317718506 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.807497262954712 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.8078553676605225 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.8080432415008545 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.8081703186035156 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.808042287826538 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.8079426288604736 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.807534694671631 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.807810068130493 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.8074188232421875 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.808187484741211 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.8083038330078125 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.8081042766571045 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.8080196380615234 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.807197332382202 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.807725429534912 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.8080177307128906 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.807894229888916 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.808204174041748 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.8082449436187744 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.808345317840576 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.8079893589019775 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.8074045181274414 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.8075459003448486 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.8082475662231445 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.8079869747161865 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.8080451488494873 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.808121919631958 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.8078136444091797 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.8079090118408203 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.8079607486724854 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.8082540035247803 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.8080697059631348 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.807921886444092 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.8078126907348633 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.808096170425415 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.807980537414551 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.8076319694519043 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.808072566986084 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.8081181049346924 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.8080267906188965 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.8082401752471924 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.8080008029937744 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.807992696762085 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.807969093322754 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.8081305027008057 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.808209180831909 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.8080666065216064 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.808128833770752 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.8079445362091064 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.8078653812408447 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.8078818321228027 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.8080806732177734 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.8082337379455566 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.808166265487671 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.8079352378845215 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.8078484535217285 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.807610273361206 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.807180881500244 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.8080039024353027 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.808074951171875 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.808108329772949 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.808115005493164 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.8080878257751465 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.8080530166625977 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.8075644969940186 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.808206796646118 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.8081536293029785 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.80812406539917 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.8081376552581787 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.8079371452331543 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.8077518939971924 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.807384967803955 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.8081586360931396 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.808220386505127 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.8080389499664307 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.8082222938537598 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.8076484203338623 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.807774543762207 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.8077948093414307 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.808037519454956 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.80818772315979 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.807918071746826 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.8078038692474365 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.807699680328369 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.8079044818878174 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.8080883026123047 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.808169364929199 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.808145523071289 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.8081679344177246 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.8080904483795166 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.8079380989074707 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.8077449798583984 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.808234930038452 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.8082377910614014 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.8081324100494385 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.8081791400909424 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.8081326484680176 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.8076655864715576 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.808016300201416 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.8079771995544434 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.808234930038452 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.808184862136841 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.808145523071289 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.808103322982788 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.8078699111938477 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.807769536972046 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.808204174041748 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.8081915378570557 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.8081157207489014 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.8082633018493652 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.808134078979492 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.808100461959839 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.8081514835357666 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.8073933124542236 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.8081130981445312 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.8080551624298096 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.8081448078155518 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.8081541061401367 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.8082399368286133 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.8079066276550293 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.808340311050415 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.808115005493164 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.8081514835357666 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.807999849319458 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.80814266204834 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.807943820953369 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.808342933654785 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.807995080947876 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.808147668838501 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.8082032203674316 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.8081796169281006 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.8080942630767822 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.8078761100769043 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.8080008029937744 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.80800199508667 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.8080646991729736 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.808115005493164 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.8080196380615234 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.808048963546753 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.8077845573425293 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.8078696727752686 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.8077609539031982 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.808087110519409 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.8082194328308105 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.807736396789551 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.808171033859253 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.807950973510742 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.808037519454956 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.80802059173584 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.808169364929199 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.8081586360931396 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.807708501815796 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.8080596923828125 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.8077805042266846 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.8077499866485596 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.8070266246795654 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.8082449436187744 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.8080453872680664 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.8012454509735107 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.8080098628997803 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.8076367378234863 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.809143543243408 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.8070991039276123 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.8081068992614746 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.8082058429718018 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.807506799697876 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.8077824115753174 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.8074958324432373 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.8080532550811768 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.8078699111938477 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.8080615997314453 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.808040142059326 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.806612730026245 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.8077008724212646 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.8080079555511475 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.80769944190979 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.808004856109619 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.8081610202789307 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.808457374572754 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.807922840118408 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.807821750640869 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.807264566421509 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.8084652423858643 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.8080246448516846 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.808131456375122 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.809450149536133 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.803058624267578 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.807701587677002 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.807703733444214 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.8082151412963867 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.8080694675445557 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.808245897293091 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.808826446533203 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.8074400424957275 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.8075435161590576 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.808159112930298 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.808025360107422 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.808356761932373 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.808208465576172 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.8083274364471436 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.806532859802246 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.8077025413513184 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.808074951171875 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.808098554611206 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.8087894916534424 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.8079261779785156 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.8080532550811768 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.8075921535491943 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.808056116104126 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.8077006340026855 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.8078789710998535 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.807922124862671 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.8083345890045166 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.808253049850464 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.8082566261291504 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.808180332183838 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.8073511123657227 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.8072445392608643 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.8056881427764893 + } + ] + }, + { + "bit": 13, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.7455973625183105 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7457339763641357 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.745497465133667 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.7458584308624268 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.745556592941284 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.745908737182617 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.745607376098633 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.745704174041748 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.745692014694214 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.7454922199249268 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.7458293437957764 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.7457592487335205 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.745523691177368 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.7457163333892822 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.745647668838501 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7457995414733887 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.7456917762756348 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7453179359436035 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.7451975345611572 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.745577812194824 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7456467151641846 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.7456679344177246 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.7456657886505127 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7456583976745605 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.745741367340088 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.7457547187805176 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.7455081939697266 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.745391368865967 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7456867694854736 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.745791435241699 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.745569944381714 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.7454912662506104 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.745474338531494 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.7458324432373047 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.7456583976745605 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.7458038330078125 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7457869052886963 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7455902099609375 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7455999851226807 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7456135749816895 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.7456748485565186 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.745514154434204 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.745702028274536 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.7456886768341064 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7455568313598633 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7455968856811523 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.745471715927124 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.745523452758789 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.745408296585083 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.7456605434417725 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.745622396469116 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7456109523773193 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7455837726593018 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.7456917762756348 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.7456905841827393 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.745591878890991 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7457501888275146 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.7455055713653564 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.745464563369751 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.745608329772949 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.745575189590454 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.7453582286834717 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.7459146976470947 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7455689907073975 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7457199096679688 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7454993724823 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.7456769943237305 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.7455403804779053 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.74564528465271 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.7453715801239014 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.745645761489868 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.74556565284729 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.7458341121673584 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.7454164028167725 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.745734214782715 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7457807064056396 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.7454941272735596 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.745913028717041 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.745471954345703 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.74554443359375 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.745514154434204 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.745718002319336 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7457966804504395 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.745643138885498 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.7457339763641357 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7456858158111572 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.745866298675537 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.7455921173095703 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7453854084014893 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.7455365657806396 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.745363712310791 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.7455294132232666 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7455475330352783 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.745448589324951 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7454707622528076 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.745457172393799 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.745774507522583 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.745889186859131 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7454628944396973 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7454819679260254 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.745779514312744 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.745610475540161 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.7451796531677246 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.7457995414733887 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7455263137817383 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7457339763641357 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.745750665664673 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7453885078430176 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.7457449436187744 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.7456648349761963 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.7457618713378906 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7458744049072266 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.7454073429107666 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.74550461769104 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7458672523498535 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.745234966278076 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.7456061840057373 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7456467151641846 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.7449734210968018 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.745706081390381 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.7456448078155518 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.7453742027282715 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.7458503246307373 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.745635986328125 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7456161975860596 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.745511054992676 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.745527505874634 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7458605766296387 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7453854084014893 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.7453975677490234 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.74558162689209 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7453811168670654 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.745678424835205 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7455599308013916 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.745530366897583 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.7454237937927246 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7457449436187744 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7453689575195312 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.745429754257202 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.7459170818328857 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.7455029487609863 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.745424509048462 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.74540638923645 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.7454588413238525 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.745370864868164 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7457404136657715 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.745701789855957 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7456367015838623 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7450759410858154 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7447164058685303 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7460639476776123 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.7455921173095703 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.74564528465271 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.745638608932495 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.74538516998291 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.7453770637512207 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.7446322441101074 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.7453372478485107 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.745647668838501 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.745581865310669 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.745450019836426 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.745389461517334 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.745405435562134 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.7451016902923584 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.745734214782715 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.7451260089874268 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.746117353439331 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.74526047706604 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.7454042434692383 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.745549440383911 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7455661296844482 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.7457282543182373 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.7455787658691406 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.745471239089966 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.7458574771881104 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.7454841136932373 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7456135749816895 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.745784282684326 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7455999851226807 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.7456233501434326 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7453341484069824 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.745460033416748 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.745755672454834 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7457475662231445 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.745493173599243 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.745753049850464 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7455503940582275 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.745542287826538 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.7457449436187744 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.745344877243042 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.74548602104187 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7430248260498047 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.745511054992676 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.7453243732452393 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7455577850341797 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.745438575744629 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.7456352710723877 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.745459794998169 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7456305027008057 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.745450973510742 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.7456302642822266 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.7455246448516846 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7455039024353027 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.7455215454101562 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.745485305786133 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.745556592941284 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7454769611358643 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.7449357509613037 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.745532512664795 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.745488166809082 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.74550461769104 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.745492458343506 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7455575466156006 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.745513439178467 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.7452211380004883 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.7453644275665283 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7452569007873535 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.7452783584594727 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.7454769611358643 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.745619058609009 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.745469808578491 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7455973625183105 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.745588779449463 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.745360851287842 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.7455310821533203 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.7455055713653564 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.745875597000122 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.745217800140381 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.7451775074005127 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.7453036308288574 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.745797634124756 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7454957962036133 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.745518207550049 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.7454476356506348 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.7456769943237305 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.745755672454834 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.7455365657806396 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.745802879333496 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.745553731918335 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.745513439178467 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7460951805114746 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.745426893234253 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7451682090759277 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7457101345062256 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.7454922199249268 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7457239627838135 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.745657444000244 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.745128870010376 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7455897331237793 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.7456655502319336 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7456135749816895 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7460062503814697 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.745382308959961 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.745609998703003 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.745178461074829 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.745558500289917 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.7453420162200928 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.7454659938812256 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.7451653480529785 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.7452895641326904 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7455296516418457 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.745617151260376 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.7455430030822754 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.7452337741851807 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.745605230331421 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.745941400527954 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.745746374130249 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.745438575744629 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.745389461517334 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.745485305786133 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7454144954681396 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.745298385620117 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.74566388130188 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.745387315750122 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.745570182800293 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.745647668838501 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.745431900024414 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.7450079917907715 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.7457456588745117 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.7455191612243652 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.745626211166382 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.745680570602417 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.7456254959106445 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.745393991470337 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.745081901550293 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7453386783599854 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7454819679260254 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.7455263137817383 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.7455368041992188 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7455263137817383 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.745434045791626 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.745659351348877 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7458865642547607 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.74548602104187 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.745514392852783 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.745579719543457 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7458252906799316 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.7454309463500977 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.7453417778015137 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.7454566955566406 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.745370864868164 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7456929683685303 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7456161975860596 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.7456414699554443 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.74580979347229 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.7455456256866455 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7451746463775635 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.745331048965454 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.745620012283325 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.745607376098633 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.745718002319336 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.7453861236572266 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.745070695877075 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.745464563369751 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.745375156402588 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.7456576824188232 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.7457218170166016 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.745647668838501 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.7455618381500244 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.7458646297454834 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.745307683944702 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.74483585357666 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7455928325653076 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.745558977127075 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7454652786254883 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.7457432746887207 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7456421852111816 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.74558687210083 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7457573413848877 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7456815242767334 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.74564528465271 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.7456917762756348 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7455599308013916 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.745607376098633 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.7452921867370605 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.745588779449463 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.745631217956543 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.745692729949951 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7456507682800293 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.745551824569702 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.745481252670288 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.7456815242767334 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.745337724685669 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.745581865310669 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7455508708953857 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7457573413848877 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.7458269596099854 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.7460477352142334 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7455387115478516 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.745548725128174 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7455222606658936 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.745487928390503 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7456209659576416 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.745702028274536 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.746044635772705 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.74568772315979 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7457404136657715 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.745715379714966 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.745558738708496 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.745607852935791 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.7453372478485107 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.745290756225586 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.745603084564209 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.745551586151123 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7455155849456787 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.745614528656006 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.745718002319336 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.7457029819488525 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7454566955566406 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.745184898376465 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7453179359436035 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.745722532272339 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7455575466156006 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7455930709838867 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.745664596557617 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.745339870452881 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.745460033416748 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.74519944190979 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.74568247795105 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7455971240997314 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.745426893234253 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.7454757690429688 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.745225191116333 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7451279163360596 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.7457828521728516 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.745652914047241 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.745645761489868 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7456905841827393 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.745417594909668 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.745091199874878 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.7452938556671143 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.7455368041992188 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.7456037998199463 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.7456440925598145 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.74556565284729 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7455830574035645 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.745631217956543 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7457563877105713 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.7459170818328857 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7458150386810303 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7456271648406982 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.745620012283325 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.745594024658203 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7454819679260254 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7454652786254883 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.745407819747925 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.745587110519409 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.7456717491149902 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.745776891708374 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.7457189559936523 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.74554181098938 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.74550461769104 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.7456748485565186 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.745793104171753 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.745628595352173 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.745553731918335 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7455806732177734 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.745617151260376 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.7456448078155518 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.7456021308898926 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.745702028274536 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.745562791824341 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.745778799057007 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7456705570220947 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.745514392852783 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.7455556392669678 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.745288133621216 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7456345558166504 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7455718517303467 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.7456908226013184 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.7455599308013916 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.7453219890594482 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.7452409267425537 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.74544358253479 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.745645761489868 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.7456088066101074 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7455546855926514 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7455806732177734 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.745436668395996 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7449796199798584 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7454962730407715 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.745728015899658 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.74562406539917 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.745626211166382 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.7456095218658447 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.74540376663208 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.7456719875335693 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.7452194690704346 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.7456305027008057 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.745696783065796 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7454590797424316 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.7456886768341064 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.7454428672790527 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7453720569610596 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.745922088623047 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.745569944381714 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7456912994384766 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.7456395626068115 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.7456390857696533 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.745231866836548 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.7456295490264893 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.7453699111938477 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.745628595352173 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.7454872131347656 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7456958293914795 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.745661497116089 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.745328426361084 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.744922161102295 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.745258331298828 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.7456717491149902 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7455549240112305 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.745614528656006 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.745640516281128 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.745603322982788 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.745382308959961 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7454476356506348 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.745565891265869 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.7457425594329834 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.7456815242767334 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.7455670833587646 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.7452306747436523 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.745412588119507 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.7457666397094727 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7457010746002197 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.7456130981445312 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.745746612548828 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7456068992614746 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.7453296184539795 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.745469808578491 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.745192289352417 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.7456493377685547 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.745600938796997 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.745587110519409 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7453207969665527 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.745337963104248 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7459468841552734 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.745793581008911 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.7455527782440186 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.7456002235412598 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.733048677444458 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.7455503940582275 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.744987726211548 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7463185787200928 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.746061086654663 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7454278469085693 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7455034255981445 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.7435109615325928 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7454662322998047 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7451653480529785 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7458202838897705 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.7454869747161865 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.7454750537872314 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.745683193206787 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.7433760166168213 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.74541974067688 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.745262622833252 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.745687246322632 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.745431423187256 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.745635509490967 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.745651960372925 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.744262218475342 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.745396614074707 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7448501586914062 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.745514392852783 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7450506687164307 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7474312782287598 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.7464406490325928 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.737316131591797 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7454419136047363 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7451765537261963 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.746032476425171 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.74568772315979 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.7458016872406006 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7464542388916016 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7452468872070312 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.74540114402771 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.7453346252441406 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.745582342147827 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.7451136112213135 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.7454028129577637 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.745762586593628 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.744636297225952 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.74566650390625 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.7455880641937256 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7458081245422363 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.7452313899993896 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7456421852111816 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7453832626342773 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.7454488277435303 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.7452590465545654 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.745265245437622 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.746155261993408 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.745635986328125 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.745741367340088 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7457797527313232 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.7456727027893066 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7456302642822266 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7453813552856445 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.745291233062744 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.744920492172241 + } + ] + }, + { + "bit": 14, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.6904873847961426 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.6905934810638428 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.6906323432922363 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.6904296875 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.6905694007873535 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.690582513809204 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.6904969215393066 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.6903820037841797 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.6904783248901367 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.690617322921753 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.6906344890594482 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.6903223991394043 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.690481424331665 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.6905200481414795 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.6904456615448 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.6905925273895264 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.690660238265991 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.690484046936035 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.6905837059020996 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.6905786991119385 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.6905908584594727 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.6904475688934326 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.6903910636901855 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.6904871463775635 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.6903955936431885 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.6900665760040283 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.6904983520507812 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.690401792526245 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.6902925968170166 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.690354108810425 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.690380573272705 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.690502166748047 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.690617322921753 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.6906464099884033 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.690380573272705 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.6905570030212402 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.6905925273895264 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.690570592880249 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.6904897689819336 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.690598487854004 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.6903340816497803 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.6906065940856934 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.6905517578125 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.6905152797698975 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.6906015872955322 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.6904802322387695 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.690384864807129 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.690565586090088 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.690706968307495 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.6902055740356445 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.6902410984039307 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.690380096435547 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.6906821727752686 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.6902647018432617 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.690528631210327 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.6904444694519043 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.6905505657196045 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.6906540393829346 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.6906661987304688 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.690525770187378 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.690584421157837 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.6902694702148438 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.690277576446533 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.690426826477051 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.6904067993164062 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.6905758380889893 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.6903979778289795 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.690570592880249 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.690369129180908 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.6903889179229736 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.690420389175415 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.6905553340911865 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.690460681915283 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.6907176971435547 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.690479278564453 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.690391778945923 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.6906421184539795 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.6904923915863037 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.6905300617218018 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.6905038356781006 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.6905934810638428 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.690321445465088 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.690444231033325 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.6902916431427 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.690553665161133 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.6906120777130127 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.6905038356781006 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.6905317306518555 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.690682888031006 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.6908469200134277 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.6907124519348145 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.6904611587524414 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.6903929710388184 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.6904008388519287 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.6904237270355225 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.690638303756714 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.6904444694519043 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.690269708633423 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.690500259399414 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.690451145172119 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.6905689239501953 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.6903295516967773 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.690765142440796 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.6903746128082275 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.6904914379119873 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.6905813217163086 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.6906161308288574 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.6902389526367188 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.690335750579834 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.6907124519348145 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.6904122829437256 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.69047212600708 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.6905670166015625 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.690345287322998 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.69075345993042 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.6907527446746826 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.690349817276001 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.690322160720825 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.690674304962158 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.6905179023742676 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.690258502960205 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.6903135776519775 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.6905038356781006 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.6906661987304688 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.690561532974243 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.6904969215393066 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.690464496612549 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.6903910636901855 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.69043231010437 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.6908998489379883 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.690526247024536 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.6906075477600098 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.690335750579834 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.690484046936035 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.6904377937316895 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.690031051635742 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.6903131008148193 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.690535545349121 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.6904139518737793 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.690319299697876 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.6905276775360107 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.6905736923217773 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.690685987472534 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.6904754638671875 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.6904187202453613 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.6905038356781006 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.690605401992798 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.69044828414917 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.6902434825897217 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.6903560161590576 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.6898958683013916 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.6901941299438477 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.690451145172119 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.6905767917633057 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.690420389175415 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.6904425621032715 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.6906216144561768 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.690474033355713 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.6903648376464844 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.6905221939086914 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.690859079360962 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.6904923915863037 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.690528631210327 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.6897361278533936 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.690189838409424 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.6906421184539795 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.690431594848633 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.6905667781829834 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.6905038356781006 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.690509080886841 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.6903388500213623 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.689976692199707 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.6902456283569336 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.690401792526245 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.6901888847351074 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.6904423236846924 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.6903750896453857 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.6903011798858643 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.6904993057250977 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.6904139518737793 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.690211772918701 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.690490484237671 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.6903789043426514 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.6904544830322266 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.6903789043426514 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.690185785293579 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.6900784969329834 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.6904892921447754 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.690438747406006 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.6904361248016357 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.6905181407928467 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.6917715072631836 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.690243721008301 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.690502405166626 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.6905152797698975 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.6906533241271973 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.6906890869140625 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.6905288696289062 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.6897807121276855 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.6904115676879883 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.690214157104492 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.690371036529541 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.6903398036956787 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.690520763397217 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.6906285285949707 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.6904144287109375 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.6903023719787598 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.690540075302124 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.690523624420166 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.6902904510498047 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.690523147583008 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.690518856048584 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.6904337406158447 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.690538167953491 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.6905601024627686 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.690384864807129 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.690443277359009 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.6904215812683105 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.6904215812683105 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.6902050971984863 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.690093755722046 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.690613031387329 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.6904518604278564 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.6906301975250244 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.690382480621338 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.6905300617218018 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.689734935760498 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.6904475688934326 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.6905629634857178 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.6904773712158203 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.690429449081421 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.690436840057373 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.6903998851776123 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.690392017364502 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.690281391143799 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.6900689601898193 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.6904354095458984 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.690460681915283 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.690574884414673 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.6905126571655273 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.6894421577453613 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.6906721591949463 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.6904890537261963 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.690281391143799 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.690103530883789 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.69044828414917 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.690207004547119 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.6904118061065674 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.6904923915863037 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.6903223991394043 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.69079327583313 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.690099000930786 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.6903350353240967 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.6905362606048584 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.6901228427886963 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.690162420272827 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.6906261444091797 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.6904139518737793 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.6906073093414307 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.6903176307678223 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.690640449523926 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.6901679039001465 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.6902995109558105 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.6903581619262695 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.690075397491455 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.690336227416992 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.690366744995117 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.6904711723327637 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.6903955936431885 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.6904377937316895 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.690546989440918 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.6903154850006104 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.690035820007324 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.6904542446136475 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.690545082092285 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.6903679370880127 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.690513849258423 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.690650224685669 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.690189838409424 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.6905529499053955 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.6903798580169678 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.690554618835449 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.690441608428955 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.690643548965454 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.690509080886841 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.690566301345825 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.6905593872070312 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.6905908584594727 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.690486192703247 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.6904969215393066 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.690546989440918 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.6901562213897705 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.690079689025879 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.690200090408325 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.690484046936035 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.6904444694519043 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.690354108810425 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.690458059310913 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.6903975009918213 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.6903865337371826 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.6903746128082275 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.690443515777588 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.690551996231079 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.690591812133789 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.6902806758880615 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.6903603076934814 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.690633535385132 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.6902170181274414 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.69045352935791 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.69044828414917 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.6902647018432617 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.6906182765960693 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.6909420490264893 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.6901967525482178 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.690110445022583 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.6904804706573486 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.6904304027557373 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.690436840057373 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.690591335296631 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.689929485321045 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.6904423236846924 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.6911120414733887 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.6906018257141113 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.6904897689819336 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.6905441284179688 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.6903841495513916 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.690403461456299 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.690272331237793 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.6902499198913574 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.690455436706543 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.6904590129852295 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.69045352935791 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.690427303314209 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.690286159515381 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.690225839614868 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.6904263496398926 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.6906373500823975 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.690462112426758 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.6905574798583984 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.6906304359436035 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.6903438568115234 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.690154552459717 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.690361976623535 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.690566301345825 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.690610408782959 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.690556764602661 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.690364122390747 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.690042018890381 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.6903998851776123 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.6903438568115234 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.690422773361206 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.6904826164245605 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.6905527114868164 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.69039249420166 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.6898608207702637 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.6903152465820312 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.6904611587524414 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.690459728240967 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.6904795169830322 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.690586805343628 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.6906471252441406 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.6902410984039307 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.6902081966400146 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.6901187896728516 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.69038987159729 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.690676212310791 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.690088987350464 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.6905956268310547 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.6906216144561768 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.6904711723327637 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.690537214279175 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.6906509399414062 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.690584659576416 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.6905221939086914 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.6905200481414795 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.6906578540802 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.6900815963745117 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.6902644634246826 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.690514326095581 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.690546989440918 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.690563678741455 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.6907804012298584 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.6905460357666016 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.6909077167510986 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.690106153488159 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.6905291080474854 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.6906251907348633 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.690690279006958 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.690567970275879 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.690830707550049 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.6905012130737305 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.690119743347168 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.690678834915161 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.690584659576416 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.690648078918457 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.6904733180999756 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.690293073654175 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.690192937850952 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.690255641937256 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.6906609535217285 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.6906886100769043 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.690600633621216 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.6904444694519043 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.6905016899108887 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.6902916431427 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.690486192703247 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.690524101257324 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.6905767917633057 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.6905016899108887 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.6905088424682617 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.6903791427612305 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.6903977394104004 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.6902358531951904 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.690600633621216 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.6906282901763916 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.690664052963257 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.690608501434326 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.6903481483459473 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.690420389175415 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.690242052078247 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.6906509399414062 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.6906216144561768 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.6904993057250977 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.6904852390289307 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.6902365684509277 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.690347194671631 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.6905996799468994 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.6906015872955322 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.6906304359436035 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.6905758380889893 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.6906261444091797 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.690652370452881 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.6903979778289795 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.6904940605163574 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.690549612045288 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.6905245780944824 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.6906304359436035 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.6906840801239014 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.690222978591919 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.6908152103424072 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.690539836883545 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.6906206607818604 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.6905934810638428 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.690521240234375 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.690439462661743 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.6903252601623535 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.690077066421509 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.6905250549316406 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.6905758380889893 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.690570592880249 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.6905734539031982 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.6905972957611084 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.6905782222747803 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.690369129180908 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.6900436878204346 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.6906216144561768 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.69063663482666 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.690514326095581 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.6905856132507324 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.690735340118408 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.6900839805603027 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.690722942352295 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.690656900405884 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.6906111240386963 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.6904165744781494 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.6904497146606445 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.6904337406158447 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.690502166748047 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.6905908584594727 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.690633535385132 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.6905670166015625 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.6905767917633057 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.6905250549316406 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.690394163131714 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.6903586387634277 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.6907782554626465 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.6905081272125244 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.690638303756714 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.6905627250671387 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.690443515777588 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.6906533241271973 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.690451145172119 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.690047025680542 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.6906423568725586 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.6905407905578613 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.689837694168091 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.690462350845337 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.6905994415283203 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.6900508403778076 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.690561532974243 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.6905055046081543 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.6905570030212402 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.6896603107452393 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.6904361248016357 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.6903955936431885 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.690251350402832 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.689913511276245 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.6905100345611572 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.6904067993164062 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.6825547218322754 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.690519094467163 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.689710855484009 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.6909639835357666 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.6899545192718506 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.6905059814453125 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.6906630992889404 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.690096616744995 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.690370559692383 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.6903131008148193 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.690577745437622 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.6907057762145996 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.690574884414673 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.690629482269287 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.6891472339630127 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.690303325653076 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.690446615219116 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.690220594406128 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.6906092166900635 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.690600633621216 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.690765142440796 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.6900806427001953 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.690402030944824 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.6899220943450928 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.6906304359436035 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.6902477741241455 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.690441370010376 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.691582441329956 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.6797935962677 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.6905837059020996 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.6900060176849365 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.6907436847686768 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.6907105445861816 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.6907219886779785 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.691129446029663 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.6896486282348633 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.6903438568115234 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.6907942295074463 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.6906464099884033 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.691681146621704 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.690678358078003 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.6908578872680664 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.689216136932373 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.6903440952301025 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.6906707286834717 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.690539836883545 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.690788507461548 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.6905181407928467 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.6907079219818115 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.690220594406128 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.690551996231079 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.6904611587524414 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.6907026767730713 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.691021680831909 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.6907060146331787 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.6907360553741455 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.6906075477600098 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.6906423568725586 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.6900153160095215 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.6900532245635986 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.6888246536254883 + } + ] + }, + { + "bit": 15, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.6355769634246826 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.635460138320923 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.635460138320923 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.6354784965515137 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.6354479789733887 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.6355206966400146 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.635586977005005 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.635493040084839 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.635592222213745 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.6354072093963623 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.6355295181274414 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.6354432106018066 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.635484218597412 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.63556170463562 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.635435104370117 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.6355743408203125 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.6355488300323486 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.6356093883514404 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.635672092437744 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.6354176998138428 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.6357057094573975 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.635485887527466 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.6357038021087646 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.6354548931121826 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.635636806488037 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.6355478763580322 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.635453224182129 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.635389566421509 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.6355702877044678 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.635359764099121 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.6356563568115234 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.6354939937591553 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.6354150772094727 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.635462760925293 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.6354012489318848 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.635575532913208 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.635403871536255 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.6353304386138916 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.635612726211548 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.6354587078094482 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.635446071624756 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.6354048252105713 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.6355814933776855 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.635645627975464 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.6355252265930176 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.6354176998138428 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.635457754135132 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.635260820388794 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.635366678237915 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.635380506515503 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.635401725769043 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.635425329208374 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.635550022125244 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.6356444358825684 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.6354548931121826 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.6355583667755127 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.635488986968994 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.635361909866333 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.6355068683624268 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.635432481765747 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.6354763507843018 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.6354684829711914 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.6356499195098877 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.6354880332946777 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.635735034942627 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.635467529296875 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.635540008544922 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.635429859161377 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.6357133388519287 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.6355745792388916 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.6356194019317627 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.635591983795166 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.6355860233306885 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.635392427444458 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.6356639862060547 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.6355793476104736 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.635580062866211 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.6355204582214355 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.6354479789733887 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.6353862285614014 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.6354050636291504 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.6355419158935547 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.6354711055755615 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.6353700160980225 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.6355643272399902 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.6358025074005127 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.635571241378784 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.635366916656494 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.6353607177734375 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.6356301307678223 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.6356751918792725 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.6355106830596924 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.635546922683716 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.63551664352417 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.635540246963501 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.635420799255371 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.6357228755950928 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.6357133388519287 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.6356008052825928 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.6356375217437744 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.635462760925293 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.63559627532959 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.635497570037842 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.635767936706543 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.6356847286224365 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.635462760925293 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.6355841159820557 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.6357505321502686 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.635364055633545 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.635608434677124 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.6356544494628906 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.6354756355285645 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.6354706287384033 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.6355693340301514 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.6355459690093994 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.635202169418335 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.635573625564575 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.6355037689208984 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.6353423595428467 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.6354072093963623 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.6354706287384033 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.6355020999908447 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.635514736175537 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.635498285293579 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.6355350017547607 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.6357977390289307 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.6354775428771973 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.6354336738586426 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.6356163024902344 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.6353724002838135 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.635491371154785 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.6355154514312744 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.635657787322998 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.6354806423187256 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.6354293823242188 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.6353235244750977 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.635540246963501 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.635474443435669 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.6355702877044678 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.635793924331665 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.6356465816497803 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.635540008544922 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.6346421241760254 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.635563373565674 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.6354360580444336 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.635805368423462 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.635375738143921 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.635754108428955 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.635309934616089 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.6349968910217285 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.6358957290649414 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.635413646697998 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.6354432106018066 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.6356563568115234 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.6352827548980713 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.6356544494628906 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.6346628665924072 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.635403871536255 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.635289430618286 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.635526180267334 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.635113477706909 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.6354525089263916 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.635497570037842 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.635214328765869 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.6358141899108887 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.6352486610412598 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.6356420516967773 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.6356613636016846 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.635688066482544 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.6356396675109863 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.635152578353882 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.6357672214508057 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.635648727416992 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.6354594230651855 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.6358070373535156 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.6356072425842285 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.6355528831481934 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.6354637145996094 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.6353023052215576 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.635593891143799 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.6357176303863525 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.6355812549591064 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.635495901107788 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.635542631149292 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.6356258392333984 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.635488986968994 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.6357152462005615 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.635805368423462 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.6353960037231445 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.6354057788848877 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.63547420501709 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.633317470550537 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.6355061531066895 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.635308027267456 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.6356778144836426 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.6357064247131348 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.635629177093506 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.635617256164551 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.6355388164520264 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.635652780532837 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.635571241378784 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.635484218597412 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.6355693340301514 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.6354684829711914 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.635348320007324 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.635294198989868 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.6351819038391113 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.635469675064087 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.6353909969329834 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.6355693340301514 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.6354360580444336 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.63558292388916 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.6350762844085693 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.635713577270508 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.635277509689331 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.6356258392333984 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.6356327533721924 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.6353466510772705 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.635593891143799 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.6356360912323 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.635664701461792 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.635488271713257 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.6355443000793457 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.6353847980499268 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.6355173587799072 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.635439395904541 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.636199951171875 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.6354334354400635 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.635467290878296 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.6354949474334717 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.6358044147491455 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.6356475353240967 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.635427474975586 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.6352455615997314 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.635573148727417 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.6358642578125 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.6355860233306885 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.6353824138641357 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.6354570388793945 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.635476589202881 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.635684013366699 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.6353814601898193 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.635446310043335 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.6355364322662354 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.635733127593994 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.635495185852051 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.63558292388916 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.635298252105713 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.63537859916687 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.635627508163452 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.6355323791503906 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.635546922683716 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.6354188919067383 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.6354429721832275 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.6353328227996826 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.635892629623413 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.63523530960083 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.6356770992279053 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.6354949474334717 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.6355643272399902 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.635364055633545 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.635483503341675 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.635526418685913 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.6353979110717773 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.6357603073120117 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.635366678237915 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.6353631019592285 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.6355109214782715 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.6354150772094727 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.6354117393493652 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.6354236602783203 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.635344982147217 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.635645627975464 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.6356496810913086 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.635509729385376 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.6354732513427734 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.6354246139526367 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.635335683822632 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.6355819702148438 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.635592222213745 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.6355156898498535 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.635469675064087 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.635368585586548 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.635488986968994 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.6353893280029297 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.635350227355957 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.635523557662964 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.635443925857544 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.6355769634246826 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.6356096267700195 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.6354072093963623 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.635735511779785 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.635650873184204 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.6353023052215576 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.635450839996338 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.635558843612671 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.6357479095458984 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.6353814601898193 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.635498046875 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.6353919506073 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.635162115097046 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.6354727745056152 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.635406970977783 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.6355068683624268 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.6355130672454834 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.6353042125701904 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.6353063583374023 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.635749578475952 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.6354312896728516 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.635479211807251 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.6355955600738525 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.6354072093963623 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.6351120471954346 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.6354501247406006 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.635568618774414 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.6354706287384033 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.635443925857544 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.635514736175537 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.635462760925293 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.6356582641601562 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.6353352069854736 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.6350009441375732 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.6355364322662354 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.635387420654297 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.6354968547821045 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.6355721950531006 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.6350719928741455 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.6357316970825195 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.635409116744995 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.6355173587799072 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.635439395904541 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.635486602783203 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.635493755340576 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.6355020999908447 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.6354167461395264 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.6353471279144287 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.635596990585327 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.635636806488037 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.635399341583252 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.6355204582214355 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.6355862617492676 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.6355087757110596 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.6351351737976074 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.6354870796203613 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.635633945465088 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.635488271713257 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.6354832649230957 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.6356518268585205 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.635519504547119 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.6354291439056396 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.6355247497558594 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.635504961013794 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.6354243755340576 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.635380506515503 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.6356921195983887 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.6352972984313965 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.6352667808532715 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.6354756355285645 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.6356430053710938 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.6354663372039795 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.6352784633636475 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.6354176998138428 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.6354820728302 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.6355223655700684 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.635420560836792 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.635359287261963 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.6356163024902344 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.6352131366729736 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.6354057788848877 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.6353814601898193 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.6351561546325684 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.6356639862060547 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.6355555057525635 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.6354546546936035 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.6355044841766357 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.6351845264434814 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.6354002952575684 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.6352288722991943 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.6354823112487793 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.6354448795318604 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.6354281902313232 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.635383129119873 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.635159492492676 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.635422945022583 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.6356112957000732 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.6355137825012207 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.6354048252105713 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.6355583667755127 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.6354403495788574 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.6349852085113525 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.6353089809417725 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.6352953910827637 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.6354751586914062 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.6355488300323486 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.635467290878296 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.635660409927368 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.635411024093628 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.63545560836792 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.6355135440826416 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.635618209838867 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.635413646697998 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.6355373859405518 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.635498285293579 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.6352975368499756 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.635446310043335 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.6351113319396973 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.6355202198028564 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.6354873180389404 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.635488271713257 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.635439157485962 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.6353561878204346 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.63521409034729 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.635402202606201 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.6355206966400146 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.635441541671753 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.6355044841766357 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.6354637145996094 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.6355597972869873 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.635593891143799 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.6356382369995117 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.6355538368225098 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.6354923248291016 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.635586977005005 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.635521411895752 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.635618209838867 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.6355762481689453 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.6356008052825928 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.635467290878296 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.63551664352417 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.635619640350342 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.6355955600738525 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.635244369506836 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.6354246139526367 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.635364532470703 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.6355676651000977 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.635499954223633 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.6354329586029053 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.635481119155884 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.6354942321777344 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.6349127292633057 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.635165214538574 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.6355605125427246 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.6355037689208984 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.635505437850952 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.6355643272399902 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.6353747844696045 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.635924816131592 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.6353249549865723 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.6354291439056396 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.6354949474334717 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.635485887527466 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.6354751586914062 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.6352641582489014 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.635402202606201 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.635671854019165 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.6354801654815674 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.6355087757110596 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.6356165409088135 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.635526180267334 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.6352834701538086 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.6353466510772705 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.634932041168213 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.635521411895752 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.6355321407318115 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.635526180267334 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.6355583667755127 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.6352834701538086 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.6352627277374268 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.6354775428771973 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.6356093883514404 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.6353981494903564 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.6355295181274414 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.635429859161377 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.635420560836792 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.635481119155884 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.635035753250122 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.6355350017547607 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.635491371154785 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.6354150772094727 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.6353919506073 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.6354098320007324 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.635296106338501 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.6354167461395264 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.635483503341675 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.6355156898498535 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.6352946758270264 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.6354825496673584 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.6349921226501465 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.635573148727417 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.6350584030151367 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.6355295181274414 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.6352953910827637 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.6354012489318848 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.635469436645508 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.6352269649505615 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.635533094406128 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.635563850402832 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.635406732559204 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.6354706287384033 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.6288001537323 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.635493040084839 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.6354870796203613 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.635976552963257 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.6357688903808594 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.635373592376709 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.635514736175537 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.6334848403930664 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.6354923248291016 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.6351845264434814 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.635460138320923 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.635420560836792 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.635484457015991 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.6357860565185547 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.633923292160034 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.6353559494018555 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.6346962451934814 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.6359050273895264 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.6355814933776855 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.635538101196289 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.6356074810028076 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.633906602859497 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.6353647708892822 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.6348769664764404 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.635664463043213 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.6353211402893066 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.6363401412963867 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.6360080242156982 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.6260018348693848 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.635244131088257 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.6351726055145264 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.635512590408325 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.6352756023406982 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.6354939937591553 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.6359071731567383 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.634483575820923 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.6352522373199463 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.6354360580444336 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.63555645942688 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.635221004486084 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.635375499725342 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.6355745792388916 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.6346824169158936 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.6355271339416504 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.6354880332946777 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.6358015537261963 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.6353390216827393 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.6356024742126465 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.6352953910827637 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.6352827548980713 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.6352386474609375 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.635354995727539 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.6357455253601074 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.6351027488708496 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.6355574131011963 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.6355807781219482 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.6354873180389404 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.6355504989624023 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.6355340480804443 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.6353814601898193 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.6353015899658203 + } + ] + } + ] +} \ No newline at end of file