diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json" new file mode 100644--- /dev/null +++ "b/reduced_ppl_average.json" @@ -0,0 +1,13520 @@ +{ + "original_ppl": 5.652626037597656, + "reduced_ppl": [ + { + "bit": 1, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": NaN + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": NaN + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": NaN + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": NaN + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": NaN + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": NaN + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": NaN + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": NaN + } + ] + }, + { + "bit": 2, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 16.04709815979004 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 16.045433044433594 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 16.487300872802734 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 15.891241073608398 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 16.010780334472656 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 16.066612243652344 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 15.91110610961914 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 15.88374137878418 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 15.772988319396973 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 16.760400772094727 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 15.517850875854492 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 15.84735107421875 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 16.013795852661133 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 15.694584846496582 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 15.974059104919434 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 15.602910995483398 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 15.7749662399292 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 15.755630493164062 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 15.608354568481445 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 15.733319282531738 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 15.620728492736816 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 15.871256828308105 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 15.709325790405273 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 15.877296447753906 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 15.738811492919922 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 15.269658088684082 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 15.550324440002441 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 15.624741554260254 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 15.790115356445312 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 15.752761840820312 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 15.435846328735352 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 15.474074363708496 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 15.38607406616211 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 15.776700019836426 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 15.762375831604004 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 15.875486373901367 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 15.99376392364502 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 15.875116348266602 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 15.883211135864258 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 15.553327560424805 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 15.763303756713867 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 15.973578453063965 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 16.075237274169922 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 16.209758758544922 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 16.08220863342285 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 16.027441024780273 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 15.816433906555176 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 15.926165580749512 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 16.103614807128906 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 16.114065170288086 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 16.021312713623047 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 15.934331893920898 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 15.984635353088379 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 15.87658405303955 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 15.974640846252441 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 15.87530517578125 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 16.148601531982422 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 16.1705265045166 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 15.888603210449219 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 15.930898666381836 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 15.994607925415039 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 15.929621696472168 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 16.007707595825195 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 16.057170867919922 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 15.943398475646973 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 15.868350982666016 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 15.971551895141602 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 15.980457305908203 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 15.834277153015137 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 16.1314697265625 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 16.070030212402344 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 16.13533592224121 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 15.85223388671875 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 15.742361068725586 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 16.077167510986328 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 16.04054832458496 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 15.981799125671387 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 16.092016220092773 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 15.977822303771973 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 16.372596740722656 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 16.048381805419922 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 16.006319046020508 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 15.899312973022461 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 16.00881004333496 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 16.120723724365234 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 16.036487579345703 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 15.827996253967285 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 15.89213752746582 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 16.13409423828125 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 16.154470443725586 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 16.10260009765625 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 16.069181442260742 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 16.039653778076172 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 15.890963554382324 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 16.009124755859375 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 16.097379684448242 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 16.068456649780273 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 16.066612243652344 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 15.984234809875488 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 16.07306480407715 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 15.970989227294922 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 16.059717178344727 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 16.06581687927246 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 16.0466365814209 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 16.033353805541992 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 16.09553337097168 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 16.065927505493164 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 16.004501342773438 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 16.085004806518555 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 16.099889755249023 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 16.0899658203125 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 16.02498435974121 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 16.065265655517578 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 16.133319854736328 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 16.127227783203125 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 16.088977813720703 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 16.10784149169922 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 16.02992820739746 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 16.06787872314453 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 16.121126174926758 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 16.091140747070312 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 16.096508026123047 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 16.089384078979492 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 16.065288543701172 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 16.057117462158203 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 16.04627799987793 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 16.093250274658203 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 16.07312774658203 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 16.078508377075195 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 16.10796546936035 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 16.043386459350586 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 16.084312438964844 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 16.089183807373047 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 16.098615646362305 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 16.112468719482422 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 16.11897087097168 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 16.086429595947266 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 16.07809829711914 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 16.04926300048828 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 16.036027908325195 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 16.112131118774414 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 16.10662078857422 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 16.087688446044922 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 16.15542221069336 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 16.062347412109375 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 16.076629638671875 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 16.049047470092773 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 16.12260627746582 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 16.123483657836914 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 16.13287353515625 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 16.11162567138672 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 16.059677124023438 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 16.05821990966797 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 16.05929183959961 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 16.115650177001953 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 16.09845542907715 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 16.15089988708496 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 16.109846115112305 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 16.07377052307129 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 16.075054168701172 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 16.054115295410156 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 16.112646102905273 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 16.116012573242188 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 16.10887908935547 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 16.10991668701172 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 16.07421112060547 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 16.09177017211914 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 16.058225631713867 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 16.135351181030273 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 16.156068801879883 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 16.154905319213867 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 16.11480140686035 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 16.09415626525879 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 16.09871482849121 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 16.067073822021484 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 16.13267707824707 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 16.145267486572266 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 16.140344619750977 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 16.13431167602539 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 16.089250564575195 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 16.09817886352539 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 16.086000442504883 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 16.102136611938477 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 16.12909698486328 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 16.120840072631836 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 16.131977081298828 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 16.10662841796875 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 16.09153938293457 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 16.08709716796875 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 16.107215881347656 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 16.110477447509766 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 16.125354766845703 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 16.10364532470703 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 16.106924057006836 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 16.085697174072266 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 16.09556770324707 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 16.095577239990234 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 16.076086044311523 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 16.137514114379883 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 16.13003921508789 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 16.124937057495117 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 16.105241775512695 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 16.097993850708008 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 16.118410110473633 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 16.125444412231445 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 16.13190460205078 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 16.129796981811523 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 16.103012084960938 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 16.07077407836914 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 16.10454750061035 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 16.115320205688477 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 16.119178771972656 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 16.151443481445312 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 16.134124755859375 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 15.968415260314941 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 15.966558456420898 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 16.056917190551758 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 16.06722640991211 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 16.099590301513672 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 16.242080688476562 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 16.146371841430664 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 16.312795639038086 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 16.46068572998047 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 15.95256233215332 + } + ] + }, + { + "bit": 3, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 7.037257194519043 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 7.033029556274414 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 7.039805889129639 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 7.023073196411133 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 7.0317535400390625 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 7.0320940017700195 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 7.020739555358887 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 7.0239362716674805 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 7.02094030380249 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 7.049227237701416 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 7.007501602172852 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 7.018913745880127 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 7.027161121368408 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 7.017128944396973 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 7.0177001953125 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 7.01645565032959 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 7.016223907470703 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 7.012224197387695 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 7.009815692901611 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 7.016254901885986 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 7.013276100158691 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 7.021862506866455 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 7.028494358062744 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 7.021991729736328 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 7.0244526863098145 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 7.006015300750732 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 7.015928268432617 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 7.01432466506958 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 7.03032112121582 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 7.031096458435059 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 7.023862361907959 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 7.021998882293701 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 7.016659259796143 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 7.0247039794921875 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 7.025598526000977 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 7.034532070159912 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 7.034703254699707 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 7.0286173820495605 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 7.025782585144043 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 7.020806312561035 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 7.025187969207764 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 7.0252203941345215 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 7.0341315269470215 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 7.0351972579956055 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 7.030771255493164 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 7.024406433105469 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 7.017670154571533 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 7.025454998016357 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 7.0252203941345215 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 7.034374713897705 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 7.038034915924072 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 7.032420635223389 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 7.0327229499816895 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 7.021120548248291 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 7.0369553565979 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 7.026325702667236 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 7.036727428436279 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 7.0347442626953125 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 7.032942771911621 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 7.033342361450195 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 7.025910377502441 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 7.029706954956055 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 7.036744594573975 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 7.031153202056885 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 7.033031463623047 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 7.027141571044922 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 7.026742935180664 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 7.030340671539307 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 7.023252487182617 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 7.034279823303223 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 7.032810688018799 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 7.034689903259277 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 7.025817394256592 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 7.0283637046813965 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 7.0324015617370605 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 7.032069683074951 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 7.030367374420166 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 7.034885406494141 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 7.033162593841553 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 7.028541564941406 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 7.031193733215332 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 7.031918525695801 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 7.031662464141846 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 7.0306854248046875 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 7.036466121673584 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 7.034590244293213 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 7.027491092681885 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 7.0286359786987305 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 7.036945819854736 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 7.0344157218933105 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 7.038798809051514 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 7.031973838806152 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 7.0305047035217285 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 7.025036334991455 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 7.027799606323242 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 7.031806945800781 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 7.031162738800049 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 7.035312652587891 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 7.03261137008667 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 7.035036087036133 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 7.02724552154541 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 7.03125 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 7.033509254455566 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 7.0309648513793945 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 7.033204555511475 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 7.034109592437744 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 7.03021240234375 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 7.028590202331543 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 7.031584739685059 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 7.033374309539795 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 7.031156063079834 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 7.035614013671875 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 7.03269100189209 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 7.031028747558594 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 7.024980068206787 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 7.030600547790527 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 7.032419681549072 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 7.030065059661865 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 7.033725261688232 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 7.033725261688232 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 7.035201072692871 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 7.032395839691162 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 7.030650615692139 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 7.027294635772705 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 7.03133487701416 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 7.029183387756348 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 7.033224582672119 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 7.03256368637085 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 7.03267240524292 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 7.0303053855896 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 7.028664588928223 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 7.031482219696045 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 7.026515960693359 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 7.032547473907471 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 7.033412933349609 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 7.033206462860107 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 7.0327067375183105 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 7.029738903045654 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 7.032529830932617 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 7.032566547393799 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 7.034861087799072 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 7.033291816711426 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 7.031905174255371 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 7.03463077545166 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 7.029331684112549 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 7.028336524963379 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 7.026121139526367 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 7.033397197723389 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 7.034199237823486 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 7.032824993133545 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 7.034746170043945 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 7.028284072875977 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 7.0303874015808105 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 7.028496265411377 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 7.032906532287598 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 7.033956050872803 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 7.034321308135986 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 7.032298564910889 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 7.029780864715576 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 7.031306743621826 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 7.030543327331543 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 7.035224914550781 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 7.034857749938965 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 7.034079551696777 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 7.036773204803467 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 7.029743194580078 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 7.029941558837891 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 7.02716588973999 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 7.037004470825195 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 7.037955284118652 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 7.035659313201904 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 7.0358405113220215 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 7.03301477432251 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 7.030012130737305 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 7.031545639038086 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 7.031369686126709 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 7.032918930053711 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 7.030515193939209 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 7.030691623687744 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 7.0325775146484375 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 7.029051780700684 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 7.02898645401001 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 7.032534122467041 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 7.0311198234558105 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 7.028702259063721 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 7.033653259277344 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 7.030984401702881 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 7.031580924987793 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 7.030720233917236 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 7.0333147048950195 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 7.034465312957764 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 7.033868789672852 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 7.034017086029053 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 7.032734394073486 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 7.033411026000977 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 7.030787944793701 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 7.031322956085205 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 7.03136682510376 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 7.035890579223633 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 7.0314130783081055 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 7.0356950759887695 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 7.028258800506592 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 7.033054828643799 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 7.033205986022949 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 7.033908367156982 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 7.035215854644775 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 7.033935070037842 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 7.0331807136535645 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 7.029027938842773 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 7.033645153045654 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 7.031917095184326 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 7.032520771026611 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 7.031520843505859 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 7.03312873840332 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 7.028571128845215 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 7.026793003082275 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 7.025079727172852 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 7.028441905975342 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 7.035344123840332 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 7.0349507331848145 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 7.031068325042725 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 7.051062107086182 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 7.065854072570801 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 7.051238536834717 + } + ] + }, + { + "bit": 4, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.221589088439941 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.22096061706543 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.21808385848999 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.219422340393066 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.219921112060547 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.2213239669799805 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.216677188873291 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.2198100090026855 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.2197771072387695 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.2164998054504395 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.2176618576049805 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.217641353607178 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.215036392211914 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.2195234298706055 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.2161431312561035 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.2186784744262695 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.214426040649414 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.216233730316162 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.217421054840088 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.215763092041016 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.2134222984313965 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.21957540512085 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.220942497253418 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.219266414642334 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.218987941741943 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.217161655426025 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.219186782836914 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.217423439025879 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.220617294311523 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.220726013183594 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.21895170211792 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.218619346618652 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.214725494384766 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.217360496520996 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.218290328979492 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.222354888916016 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.220931053161621 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.219776153564453 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.221126556396484 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.216718673706055 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.2200751304626465 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.219257354736328 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.221800804138184 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.220972537994385 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.221912860870361 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.220513343811035 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.2189226150512695 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.222107410430908 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.22226095199585 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.221396446228027 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.22243070602417 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.219455242156982 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.221556663513184 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.218535900115967 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.218740463256836 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.221077919006348 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.2232208251953125 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.221567630767822 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.220203876495361 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.219235897064209 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.219658374786377 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.221128463745117 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.221531391143799 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.221350193023682 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.220597267150879 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.2189860343933105 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.222745418548584 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.218510627746582 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.220818996429443 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.222007751464844 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.22084903717041 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.22098445892334 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.220521450042725 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.218890190124512 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.22150182723999 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.221920013427734 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.221926212310791 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.222366809844971 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.222143650054932 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.2225661277771 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.2212090492248535 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.2197771072387695 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.220979690551758 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.219299793243408 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.221197128295898 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.220927715301514 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.219721794128418 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.21952486038208 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.221332550048828 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.221494197845459 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.2219133377075195 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.22031307220459 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.219582557678223 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.218453407287598 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.217637062072754 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.219839096069336 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.221506118774414 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.222993850708008 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.221564769744873 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.221035957336426 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.218019962310791 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.218557357788086 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.221621513366699 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.220714569091797 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.2211174964904785 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.221378326416016 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.221470355987549 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.219318389892578 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.219069480895996 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.22068452835083 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.220023155212402 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.2204084396362305 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.220788478851318 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.2209038734436035 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.220269680023193 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.218261241912842 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.220208644866943 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.221064567565918 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.219566345214844 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.220645904541016 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.222212314605713 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.220055103302002 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.220407009124756 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.220695972442627 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.221329689025879 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.221039772033691 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.221803188323975 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.221297264099121 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.220623970031738 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.220828533172607 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.220691680908203 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.221120357513428 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.220875263214111 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.220763206481934 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.220782279968262 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.220531940460205 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.221286296844482 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.221534252166748 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.221012592315674 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.220373630523682 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.221921443939209 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.221277713775635 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.221165180206299 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.220356464385986 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.220224380493164 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.22003173828125 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.220373153686523 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.22186803817749 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.221657752990723 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.2218804359436035 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.221532821655273 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.219066619873047 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.220622539520264 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.221748352050781 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.220949649810791 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.220974922180176 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.2216925621032715 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.221133708953857 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.219768047332764 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.220573425292969 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.218782424926758 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.221519470214844 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.221663475036621 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.221587181091309 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.222087860107422 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.219536781311035 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.219453811645508 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.22084903717041 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.223572731018066 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.223241806030273 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.223457336425781 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.221822261810303 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.221246242523193 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.2215728759765625 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.219231128692627 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.220243453979492 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.219858646392822 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.219914436340332 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.220199108123779 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.220577716827393 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.22098445892334 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.219793319702148 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.22134256362915 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.2195634841918945 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.2215704917907715 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.220861911773682 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.221632480621338 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.221384525299072 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.2200164794921875 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.221912860870361 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.221158981323242 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.223333835601807 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.222128868103027 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.221458911895752 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.220625400543213 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.217928409576416 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.219460964202881 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.2195143699646 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.222113132476807 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.221070289611816 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.221309185028076 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.221428871154785 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.219520568847656 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.221397876739502 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.220905780792236 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.220391273498535 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.222030162811279 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.220151424407959 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.218199253082275 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.220696926116943 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.221225738525391 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.2212724685668945 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.219708442687988 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.220144748687744 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.220312595367432 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.214944362640381 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.221689701080322 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.220263957977295 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.2199273109436035 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.220920562744141 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.221547603607178 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.227534294128418 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.234374046325684 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.221222877502441 + } + ] + }, + { + "bit": 5, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.950534343719482 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.950632095336914 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.950092792510986 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.949376106262207 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.950104236602783 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.950512409210205 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.949431419372559 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.950026035308838 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.949953556060791 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.949274063110352 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.949527740478516 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.9492597579956055 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.947882652282715 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.950282573699951 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.948899745941162 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.948819160461426 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.948472499847412 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.948841571807861 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.948764801025391 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.948957443237305 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.948505878448486 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.949666976928711 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.949838638305664 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.948698043823242 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.949564456939697 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.948849678039551 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.947506427764893 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.950237274169922 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.950210094451904 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.950133323669434 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.949648380279541 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.950083255767822 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.949525833129883 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.949862003326416 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.950461387634277 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.950273036956787 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.950390815734863 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.948370456695557 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.948822975158691 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.948535919189453 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.9489521980285645 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.949394226074219 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.950187683105469 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.949131965637207 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.949024200439453 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.949711799621582 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.948880672454834 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.948459625244141 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.949564456939697 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.94985818862915 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.951333522796631 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.949055194854736 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.9486517906188965 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.9499053955078125 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.949880123138428 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.950569152832031 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.951118469238281 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.9509172439575195 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.949199199676514 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.949212074279785 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.9516520500183105 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.9487690925598145 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.950523853302002 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.950255393981934 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.950394153594971 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.950462818145752 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.948500633239746 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.9507975578308105 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.949678421020508 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.950099468231201 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.949889659881592 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.949623107910156 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.947173595428467 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.948292255401611 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.95015287399292 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.95105504989624 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.949645042419434 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.950246810913086 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.950724124908447 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.949354648590088 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.950860977172852 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.9510273933410645 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.950108051300049 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.95116662979126 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.950627326965332 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.949697971343994 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.94857931137085 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.948486804962158 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.951731204986572 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.950328826904297 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.949828624725342 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.949443817138672 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.949672698974609 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.948781490325928 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.948502063751221 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.950438976287842 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.94995641708374 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.950353145599365 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.950380325317383 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.95053243637085 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.9501166343688965 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.950367450714111 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.949220180511475 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.950743675231934 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.950081825256348 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.950155258178711 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.9505391120910645 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.949598789215088 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.950256824493408 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.949618816375732 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.951354026794434 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.949873447418213 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.950064182281494 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.950142860412598 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.949174880981445 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.949221611022949 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.949306011199951 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.949989318847656 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.950014591217041 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.95007848739624 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.9495744705200195 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.949560642242432 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.949631690979004 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.95039176940918 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.949575901031494 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.950171947479248 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.950398921966553 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.950133323669434 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.949367523193359 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.949905872344971 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.9490790367126465 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.950349807739258 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.949882984161377 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.950170516967773 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.950404167175293 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.949997425079346 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.950371742248535 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.9492292404174805 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.949821472167969 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.949634552001953 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.950519561767578 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.950290203094482 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.9501166343688965 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.9503254890441895 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.9496283531188965 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.949340343475342 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.94932222366333 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.949796199798584 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.95030403137207 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.95057487487793 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.949787616729736 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.949281215667725 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.949886798858643 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.948587417602539 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.95026159286499 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.95000696182251 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.950655460357666 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.950598239898682 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.949498176574707 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.949393272399902 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.949532508850098 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.950436115264893 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.950312614440918 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.950960159301758 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.950201988220215 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.949927806854248 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.9503607749938965 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.9495849609375 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.951110363006592 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.951583385467529 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.950979232788086 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.950188159942627 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.949831962585449 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.950302600860596 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.948937892913818 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.949855804443359 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.94970178604126 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.950349807739258 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.9492387771606445 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.949586868286133 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.949149131774902 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.949968338012695 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.950306415557861 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.950037002563477 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.947078704833984 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.950357913970947 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.949149131774902 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.949516296386719 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.949387550354004 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.950705528259277 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.950314044952393 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.950494766235352 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.950078964233398 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.950081825256348 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.949540138244629 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.950109481811523 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.948771953582764 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.949316501617432 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.950409412384033 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.949980735778809 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.9507598876953125 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.9498724937438965 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.94980525970459 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.949910640716553 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.949618816375732 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.9496355056762695 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.94971227645874 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.949416637420654 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.949386119842529 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.9506144523620605 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.949807643890381 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.950131893157959 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.949221611022949 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.9494524002075195 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.949524879455566 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.949182510375977 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.949041843414307 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.949779510498047 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.950139999389648 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.948716640472412 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.9499077796936035 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.953163146972656 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.953240871429443 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.954277515411377 + } + ] + }, + { + "bit": 6, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.830329418182373 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.830387592315674 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.830456256866455 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.830321311950684 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.830015182495117 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.830323696136475 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.829988479614258 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.83050012588501 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.830422401428223 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.829197883605957 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.830036640167236 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.8293843269348145 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.828919887542725 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.8292741775512695 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.829680442810059 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.829955577850342 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.830733776092529 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.829782485961914 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.829398155212402 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.829826831817627 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.8301801681518555 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.830142974853516 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.830225467681885 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.830151557922363 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.829891204833984 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.830938816070557 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.8308210372924805 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.828967094421387 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.830413818359375 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.8306474685668945 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.829627513885498 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.829604148864746 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.830125331878662 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.830150604248047 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.8292975425720215 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.830897808074951 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.830715179443359 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.829780101776123 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.830556869506836 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.830214977264404 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.829622745513916 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.829436302185059 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.8303375244140625 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.829964637756348 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.8294548988342285 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.829546928405762 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.830349922180176 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.831423282623291 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.830128192901611 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.831001281738281 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.831032752990723 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.829922199249268 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.830742359161377 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.8299970626831055 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.830557346343994 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.830245018005371 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.8307576179504395 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.830504894256592 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.830038547515869 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.830301284790039 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.829351425170898 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.830323696136475 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.829890727996826 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.8304266929626465 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.830495834350586 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.828789234161377 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.830775260925293 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.82969331741333 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.829888343811035 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.830686569213867 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.8302903175354 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.830450057983398 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.82914400100708 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.829426288604736 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.830687522888184 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.830323696136475 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.830966949462891 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.8311662673950195 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.831111431121826 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.829726696014404 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.830172061920166 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.8299760818481445 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.8304548263549805 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.83052921295166 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.830428123474121 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.830844402313232 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.830337047576904 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.830316543579102 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.830831050872803 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.831816673278809 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.830974578857422 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.8302459716796875 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.829955577850342 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.829085826873779 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.829522609710693 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.829666614532471 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.83113431930542 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.830230712890625 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.830348491668701 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.830618381500244 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.829896926879883 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.830080032348633 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.830712795257568 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.8300557136535645 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.8301167488098145 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.830312252044678 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.830120086669922 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.829613208770752 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.829955577850342 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.830979824066162 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.830132484436035 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.830628871917725 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.830431938171387 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.830546855926514 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.829552173614502 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.829737186431885 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.83035135269165 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.82991361618042 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.83075475692749 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.83018159866333 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.830189228057861 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.830137729644775 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.829817771911621 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.82933235168457 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.83029317855835 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.830000877380371 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.830139636993408 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.830382347106934 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.830811500549316 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.8304548263549805 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.830015182495117 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.830526828765869 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.829657554626465 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.830320835113525 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.830142974853516 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.830170154571533 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.8302693367004395 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.830021858215332 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.829999923706055 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.830141544342041 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.830631732940674 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.830875396728516 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.830021858215332 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.8302903175354 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.829461574554443 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.829933166503906 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.829598903656006 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.830214977264404 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.830302715301514 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.830825328826904 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.830611705780029 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.8299055099487305 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.829500198364258 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.830472469329834 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.830243110656738 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.83011531829834 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.830312252044678 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.8298540115356445 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.830522060394287 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.8300604820251465 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.829891204833984 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.830517768859863 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.830812454223633 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.830211162567139 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.830710411071777 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.829898357391357 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.830170631408691 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.829765796661377 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.830783843994141 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.831235408782959 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.830253601074219 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.830622673034668 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.830408096313477 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.830137729644775 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.829638481140137 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.829920768737793 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.829589366912842 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.829530239105225 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.830206871032715 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.830812931060791 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.830207824707031 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.830089569091797 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.830361843109131 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.830513000488281 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.830187797546387 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.830045700073242 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.831257343292236 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.830786228179932 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.829954624176025 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.8304877281188965 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.830513000488281 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.8306989669799805 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.8303351402282715 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.830267429351807 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.830773830413818 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.830455780029297 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.829495429992676 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.829456806182861 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.830803871154785 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.830038547515869 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.83054256439209 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.830592632293701 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.830241680145264 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.830111026763916 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.83049201965332 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.829438209533691 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.830401420593262 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.8305439949035645 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.829553604125977 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.8298444747924805 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.830380916595459 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.830348491668701 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.830320835113525 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.829876899719238 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.829224109649658 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.829258918762207 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.831153392791748 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.830034255981445 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.830395698547363 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.830821990966797 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.82942008972168 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.8316545486450195 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.835211753845215 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.832214832305908 + } + ] + }, + { + "bit": 7, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.780698776245117 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.7805962562561035 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.780807018280029 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.780706405639648 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.780838966369629 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.7806396484375 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.780917644500732 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.78062105178833 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.78048849105835 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.781406879425049 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.7815752029418945 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.780229091644287 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.7797932624816895 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.780560493469238 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.7800798416137695 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.780355930328369 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.780239582061768 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.780325889587402 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.7797770500183105 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.780242919921875 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.780238151550293 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.7807440757751465 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.780720233917236 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.780678749084473 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.780490875244141 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.7794952392578125 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.780674457550049 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.780398845672607 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.7806267738342285 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.780764102935791 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.780209064483643 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.780563831329346 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.780310153961182 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.78105354309082 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.780825614929199 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.780741214752197 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.781108856201172 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.779613971710205 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.780027866363525 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.780713081359863 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.781558513641357 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.781186580657959 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.780183792114258 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.780994892120361 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.779972076416016 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.780916213989258 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.780180931091309 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.77996301651001 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.780218601226807 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.781047344207764 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.7808098793029785 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.7805585861206055 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.780381202697754 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.781258583068848 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.780784606933594 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.779740333557129 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.780942440032959 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.781252384185791 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.780187606811523 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.78049373626709 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.781225681304932 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.780028343200684 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.780884265899658 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.7805962562561035 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.780635356903076 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.780574321746826 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.7803826332092285 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.781196117401123 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.780906677246094 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.7803874015808105 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.7804388999938965 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.780409336090088 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.779181003570557 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.780531406402588 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.780866622924805 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.781018257141113 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.780320644378662 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.781118869781494 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.780873775482178 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.781398296356201 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.781306266784668 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.780183792114258 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.781096458435059 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.780618190765381 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.780991554260254 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.780623912811279 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.780021667480469 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.780597686767578 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.780918598175049 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.780928134918213 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.781340599060059 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.780554294586182 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.78037691116333 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.7795209884643555 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.779716491699219 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.780733108520508 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.780272483825684 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.780514240264893 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.780782699584961 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.780804634094238 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.780441761016846 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.780480861663818 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.7806196212768555 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.780943393707275 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.7808356285095215 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.780684947967529 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.780652046203613 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.779844760894775 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.780035495758057 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.780524730682373 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.780982971191406 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.780488967895508 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.780518054962158 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.78066349029541 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.779298305511475 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.780218601226807 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.7800517082214355 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.781335353851318 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.780642986297607 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.780619144439697 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.780330181121826 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.780032634735107 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.780615329742432 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.780594825744629 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.780536651611328 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.780073165893555 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.780794143676758 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.780764102935791 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.780390739440918 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.780398845672607 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.780311584472656 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.780367374420166 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.7805328369140625 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.780658721923828 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.780667304992676 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.7804765701293945 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.780651092529297 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.780601978302002 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.780519962310791 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.780566692352295 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.780939102172852 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.780739784240723 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.780673503875732 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.780662536621094 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.780694007873535 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.780545234680176 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.780292987823486 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.780783176422119 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.780658721923828 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.780356407165527 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.780277252197266 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.78054666519165 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.780276775360107 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.780113697052002 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.780661106109619 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.780318737030029 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.781075954437256 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.780681610107422 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.780096054077148 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.780679702758789 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.7806525230407715 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.780791282653809 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.780770301818848 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.781324863433838 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.781072616577148 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.780580997467041 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.780710697174072 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.7804460525512695 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.781248569488525 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.7812581062316895 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.780926704406738 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.780697345733643 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.780508041381836 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.780767440795898 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.780843257904053 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.780360698699951 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.780077934265137 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.780854225158691 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.780344009399414 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.7804484367370605 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.780763149261475 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.78050422668457 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.780488967895508 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.780115127563477 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.7792582511901855 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.780689716339111 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.780311107635498 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.780444622039795 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.780447006225586 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.780757904052734 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.78084659576416 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.780725955963135 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.781230449676514 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.780903339385986 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.780115604400635 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.780249118804932 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.780117988586426 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.779996871948242 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.779958248138428 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.780666351318359 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.781451225280762 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.780857563018799 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.780367851257324 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.780457973480225 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.78033447265625 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.780634880065918 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.780550956726074 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.780287742614746 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.780181407928467 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.7809672355651855 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.7805304527282715 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.780338287353516 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.779984951019287 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.780333518981934 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.781105041503906 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.781049728393555 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.780460834503174 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.780515670776367 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.780892848968506 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.780107498168945 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.781040191650391 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.782511234283447 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.7820258140563965 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.784526348114014 + } + ] + }, + { + "bit": 8, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.7509589195251465 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.751088619232178 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.75097131729126 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.750695705413818 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.750877857208252 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.751076698303223 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.750511646270752 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.751060485839844 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.750980854034424 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.750570774078369 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.75067663192749 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.750359058380127 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.750728130340576 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.750737190246582 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.750597953796387 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.751031398773193 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.750985145568848 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.751081466674805 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.750657081604004 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.750865459442139 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.750370502471924 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.751093864440918 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.750650405883789 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.7505269050598145 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.750967979431152 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.751308441162109 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.75077486038208 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.750659942626953 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.751126289367676 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.7512078285217285 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.751067161560059 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.750854015350342 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.750921726226807 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.7502665519714355 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.7507500648498535 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.75117826461792 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.751165866851807 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.750744819641113 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.750757694244385 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.75075626373291 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.7501540184021 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.7504401206970215 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.751466751098633 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.750914096832275 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.750724792480469 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.749932289123535 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.750726699829102 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.751652717590332 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.752018451690674 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.750772953033447 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.751486778259277 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.750457286834717 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.7507405281066895 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.750732421875 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.751740455627441 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.751141548156738 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.751251220703125 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.750921726226807 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.750926494598389 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.751192569732666 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.750668048858643 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.751459121704102 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.75179386138916 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.751246929168701 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.751189231872559 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.749942779541016 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.750816345214844 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.75052547454834 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.750616073608398 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.751451015472412 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.751200199127197 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.751056671142578 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.750188827514648 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.7503581047058105 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.750874996185303 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.751070022583008 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.751440525054932 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.751589298248291 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.751640319824219 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.750307083129883 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.750836372375488 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.7510480880737305 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.750599384307861 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.7507734298706055 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.751192092895508 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.751307487487793 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.750491142272949 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.7508625984191895 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.75134801864624 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.75150728225708 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.751075267791748 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.750889301300049 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.750669002532959 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.750943660736084 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.7507734298706055 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.750864028930664 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.751318454742432 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.751629829406738 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.750824451446533 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.751153469085693 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.75085973739624 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.751075267791748 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.751059055328369 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.751065731048584 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.750549793243408 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.751150608062744 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.751128196716309 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.751029968261719 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.750865936279297 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.751530170440674 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.751400470733643 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.751282215118408 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.750977039337158 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.751020431518555 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.750988006591797 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.751034259796143 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.751100063323975 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.7506184577941895 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.751681804656982 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.750800609588623 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.751231670379639 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.750966548919678 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.7507171630859375 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.75064754486084 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.750856876373291 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.7515363693237305 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.751078128814697 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.751049995422363 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.751103401184082 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.751087665557861 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.751033306121826 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.751471519470215 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.751206874847412 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.7509307861328125 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.750874996185303 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.750908851623535 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.750967025756836 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.751197338104248 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.750866889953613 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.750896453857422 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.75127649307251 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.7512078285217285 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.751014709472656 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.7509050369262695 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.750904083251953 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.7508955001831055 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.750961780548096 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.7508368492126465 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.750876426696777 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.751278877258301 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.751194000244141 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.7507243156433105 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.750929832458496 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.751200199127197 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.750758647918701 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.7510809898376465 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.750924587249756 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.751063823699951 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.751008033752441 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.750904560089111 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.750939846038818 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.751084804534912 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.751038074493408 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.75093412399292 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.7508955001831055 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.751068592071533 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.750824451446533 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.750762462615967 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.751302719116211 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.7515716552734375 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.751301288604736 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.750890254974365 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.751097202301025 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.750977039337158 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.750679969787598 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.7507500648498535 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.750607490539551 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.750726699829102 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.750817775726318 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.751288890838623 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.750794410705566 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.751136302947998 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.751039028167725 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.750887393951416 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.751184940338135 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.750913619995117 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.750949382781982 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.751286506652832 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.750643253326416 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.7513580322265625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.751265525817871 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.751415252685547 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.750927448272705 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.751208305358887 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.75129508972168 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.7512946128845215 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.750588417053223 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.750575065612793 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.751431465148926 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.750924587249756 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.750861167907715 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.7507243156433105 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.75081205368042 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.751088619232178 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.750953197479248 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.751083850860596 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.750947952270508 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.751042366027832 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.751032829284668 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.750698566436768 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.750665664672852 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.751184940338135 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.75051736831665 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.750839710235596 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.7507524490356445 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.750371932983398 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.750448703765869 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.750528335571289 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.750783443450928 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.750861167907715 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.7503838539123535 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.751603126525879 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.75309944152832 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.751760959625244 + } + ] + }, + { + "bit": 9, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.728701591491699 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.728734493255615 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.728501319885254 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.728674411773682 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.728637218475342 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.7288103103637695 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.728559494018555 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.728695869445801 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.728588581085205 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.728524208068848 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.729066371917725 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.728916168212891 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.727735996246338 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.728409767150879 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.728429794311523 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.728477954864502 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.728076934814453 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.728283882141113 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.728623867034912 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.728069305419922 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.728968620300293 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.72851037979126 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.728906631469727 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.728947162628174 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.72907829284668 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.7279887199401855 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.728499889373779 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.728795528411865 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.728688716888428 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.72859525680542 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.728062629699707 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.728785514831543 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.728303909301758 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.72900915145874 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.729001045227051 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.728687763214111 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.7288408279418945 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.728175163269043 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.728811264038086 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.728734493255615 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.729181289672852 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.728738784790039 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.7282586097717285 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.728898525238037 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.728225231170654 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.728926658630371 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.7286272048950195 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.727887153625488 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.728799343109131 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.7289347648620605 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.728849411010742 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.728614330291748 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.728735446929932 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.729180335998535 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.729072570800781 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.728359699249268 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.728851795196533 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.728744029998779 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.728675842285156 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.728429794311523 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.7293701171875 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.728740215301514 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.728578090667725 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.728763103485107 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.72861385345459 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.728805065155029 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.728998184204102 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.72891902923584 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.72897481918335 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.728353023529053 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.7284040451049805 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.728718280792236 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.727849960327148 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.728891372680664 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.728612899780273 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.728773593902588 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.728649139404297 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.728903293609619 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.72869873046875 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.72924280166626 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.728919506072998 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.728946208953857 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.728755950927734 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.728873252868652 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.728816509246826 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.728670597076416 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.728809833526611 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.728774070739746 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.7287421226501465 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.729056358337402 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.728701114654541 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.728531837463379 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.728427886962891 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.727697849273682 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.728151321411133 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.728732109069824 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.728521347045898 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.7281999588012695 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.728780746459961 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.72865104675293 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.72816801071167 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.7286224365234375 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.728743076324463 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.728550434112549 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.729093551635742 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.728696823120117 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.72869348526001 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.7279839515686035 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.728500843048096 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.728158473968506 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.7285380363464355 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.728486061096191 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.728769302368164 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.728594779968262 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.727660655975342 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.7281813621521 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.728364944458008 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.729189395904541 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.7281389236450195 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.72857666015625 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.728541374206543 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.728137493133545 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.728636264801025 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.7286787033081055 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.72857666015625 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.728571891784668 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.72864294052124 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.728794574737549 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.728609561920166 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.728508949279785 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.728379249572754 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.728338718414307 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.728229999542236 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.728700160980225 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.728760242462158 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.7288007736206055 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.728574752807617 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.728621482849121 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.728704929351807 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.728606224060059 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.728718280792236 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.728718280792236 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.72845983505249 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.7288126945495605 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.728508949279785 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.728658676147461 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.728329658508301 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.728811264038086 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.728771209716797 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.728425979614258 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.728555679321289 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.728695392608643 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.728752613067627 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.7280707359313965 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.7285919189453125 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.72843599319458 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.728689193725586 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.728504180908203 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.728626251220703 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.7284698486328125 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.72845983505249 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.728769779205322 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.728724956512451 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.728867053985596 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.728957653045654 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.728363990783691 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.728604793548584 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.728653907775879 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.729053497314453 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.728907108306885 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.728738784790039 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.7286248207092285 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.728424549102783 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.7285966873168945 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.728694915771484 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.7285566329956055 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.728301525115967 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.728295803070068 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.728564262390137 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.728475093841553 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.7287821769714355 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.728490352630615 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.728578090667725 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.72841215133667 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.7281599044799805 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.728597164154053 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.728917598724365 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.72844123840332 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.72853946685791 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.728527069091797 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.728524208068848 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.728604793548584 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.728849411010742 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.728664398193359 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.728569030761719 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.728663444519043 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.728468418121338 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.728311538696289 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.728260517120361 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.7285590171813965 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.729045867919922 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.728965759277344 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.728293418884277 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.728581428527832 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.728546142578125 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.728402137756348 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.728529453277588 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.7282915115356445 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.728146553039551 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.728857517242432 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.728559970855713 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.728396415710449 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.728405952453613 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.728466033935547 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.728808403015137 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.727493762969971 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.728549480438232 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.728600978851318 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.728991985321045 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.728516101837158 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.72878360748291 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.729823112487793 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.729996681213379 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.73028564453125 + } + ] + }, + { + "bit": 10, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.708897590637207 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.708856582641602 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.708858013153076 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.708733081817627 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.708921909332275 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.708860874176025 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.708383083343506 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.708934307098389 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.7088704109191895 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.707984924316406 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.7086591720581055 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.708343982696533 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.708611965179443 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.708724021911621 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.708822250366211 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.708426475524902 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.709152698516846 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.708831787109375 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.708275318145752 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.708811283111572 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.7084550857543945 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.7086334228515625 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.708502769470215 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.7083916664123535 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.708404541015625 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.708988666534424 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.708848476409912 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.708823204040527 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.708720684051514 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.709097385406494 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.7090959548950195 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.708314895629883 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.709256649017334 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.708709716796875 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.708486080169678 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.709042072296143 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.70883321762085 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.708593368530273 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.709068298339844 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.708550453186035 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.708457946777344 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.708471775054932 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.708931922912598 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.7087531089782715 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.70817232131958 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.708366394042969 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.708697319030762 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.710135459899902 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.708590030670166 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.708824634552002 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.708964824676514 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.708422660827637 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.708594799041748 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.708673000335693 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.708848476409912 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.708531379699707 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.708816051483154 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.709024906158447 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.70849609375 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.709029674530029 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.708654880523682 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.709071636199951 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.70914363861084 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.7088518142700195 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.708824634552002 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.708261013031006 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.708082675933838 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.708625316619873 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.708498001098633 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.708919048309326 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.708945274353027 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.708810329437256 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.707827568054199 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.708240032196045 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.70897912979126 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.709500312805176 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.708913326263428 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.7090349197387695 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.709148406982422 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.708897590637207 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.708588600158691 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.708841323852539 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.7087531089782715 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.708553314208984 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.708844184875488 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.70893669128418 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.7085161209106445 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.70882511138916 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.708946704864502 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.708878040313721 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.709308624267578 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.708745956420898 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.708795070648193 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.708780288696289 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.708817481994629 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.7088775634765625 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.70893669128418 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.709288120269775 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.708669185638428 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.708885669708252 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.708869934082031 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.708777904510498 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.708482265472412 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.708862781524658 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.70835018157959 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.70876407623291 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.708796501159668 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.708807468414307 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.708544731140137 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.709455490112305 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.709420204162598 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.709173679351807 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.708802223205566 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.708793640136719 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.708856105804443 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.708694934844971 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.708703517913818 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.708730220794678 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.709183692932129 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.70859956741333 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.709079742431641 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.708749771118164 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.708347320556641 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.70852518081665 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.7090349197387695 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.708643913269043 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.708797931671143 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.708761215209961 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.709003448486328 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.708962440490723 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.708695888519287 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.708948612213135 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.708571910858154 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.708703517913818 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.7087225914001465 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.708823204040527 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.708759784698486 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.708735942840576 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.708853721618652 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.708878040313721 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.708869457244873 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.708891868591309 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.708773612976074 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.708921909332275 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.708657741546631 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.708622455596924 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.708792686462402 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.708544731140137 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.708776473999023 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.708747386932373 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.708725452423096 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.708456993103027 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.708620548248291 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.709471225738525 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.708795070648193 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.708745956420898 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.70867919921875 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.70890998840332 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.708883762359619 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.708775043487549 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.709000587463379 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.708817958831787 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.708889007568359 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.708724021911621 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.708648681640625 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.708697319030762 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.708805561065674 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.709014415740967 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.709143161773682 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.709136962890625 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.709082126617432 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.708853244781494 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.708871364593506 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.708769798278809 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.708428859710693 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.708650588989258 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.708488941192627 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.708700656890869 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.708616733551025 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.70889139175415 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.708786964416504 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.708803653717041 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.708795070648193 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.708676815032959 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.708736419677734 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.708659648895264 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.708780765533447 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.709181785583496 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.708777904510498 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.709079742431641 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.70894193649292 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.708888053894043 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.708797454833984 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.708899021148682 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.708863735198975 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.70903205871582 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.708508491516113 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.708353042602539 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.708913326263428 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.708691596984863 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.709018707275391 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.708836555480957 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.708751678466797 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.708701133728027 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.708832740783691 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.708528518676758 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.7088117599487305 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.70913553237915 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.708817481994629 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.70851993560791 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.70873498916626 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.708713531494141 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.708500862121582 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.708756446838379 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.708627223968506 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.709149360656738 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.708207607269287 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.708621025085449 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.708584308624268 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.708686828613281 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.708632946014404 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.709429740905762 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.709486484527588 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.7102789878845215 + } + ] + }, + { + "bit": 11, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.695955753326416 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.695981502532959 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.695910453796387 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.695834636688232 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.695790767669678 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.695825576782227 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.695971965789795 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.6957807540893555 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.6959381103515625 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.694118499755859 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.696442127227783 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.695910453796387 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.69551420211792 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.696005821228027 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.6956868171691895 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.695998668670654 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.6953535079956055 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.695550918579102 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.695770740509033 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.69602632522583 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.695738315582275 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.696061611175537 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.696165561676025 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.695980072021484 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.695896148681641 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.695885181427002 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.6960530281066895 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.695647716522217 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.695959568023682 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.695942401885986 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.695586681365967 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.6963396072387695 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.6958489418029785 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.695839881896973 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.695735454559326 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.696002006530762 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.695963382720947 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.695341110229492 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.69569730758667 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.696035861968994 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.696341514587402 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.695963382720947 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.695750713348389 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.6958394050598145 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.695704460144043 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.695923805236816 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.69585657119751 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.695278644561768 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.696381092071533 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.69588565826416 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.695934295654297 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.695746421813965 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.696000576019287 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.695972919464111 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.695716857910156 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.695772171020508 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.6961541175842285 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.696035385131836 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.6961350440979 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.69569730758667 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.696417331695557 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.695980072021484 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.695923805236816 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.695922374725342 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.695969581604004 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.6959147453308105 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.696135997772217 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.695936679840088 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.6959757804870605 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.695974349975586 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.695793628692627 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.695918083190918 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.695650577545166 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.6957902908325195 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.6958394050598145 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.6957478523254395 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.695716857910156 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.696350574493408 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.696087837219238 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.6955084800720215 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.696371555328369 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.6958184242248535 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.695952415466309 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.6964521408081055 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.696011066436768 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.69607400894165 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.695919513702393 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.695944786071777 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.6958842277526855 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.696376800537109 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.695568084716797 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.696059226989746 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.695750713348389 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.695347309112549 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.69523811340332 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.695905685424805 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.696089744567871 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.695627689361572 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.695966720581055 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.695931911468506 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.695390701293945 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.695756912231445 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.69602632522583 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.695826053619385 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.695985317230225 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.696071147918701 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.695901393890381 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.695448875427246 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.695957183837891 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.695381164550781 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.695718288421631 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.695939540863037 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.695950031280518 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.696035861968994 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.695272445678711 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.695548057556152 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.695886611938477 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.695843696594238 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.695706367492676 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.695865631103516 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.695749282836914 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.69559383392334 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.696040153503418 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.695858001708984 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.695555210113525 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.695744037628174 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.6960248947143555 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.696014881134033 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.695652961730957 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.695777893066406 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.6956892013549805 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.695688247680664 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.695781707763672 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.69591760635376 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.695920467376709 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.695887565612793 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.695928573608398 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.696014404296875 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.695777893066406 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.6958160400390625 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.696052074432373 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.696064472198486 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.695695400238037 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.695887088775635 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.6959452629089355 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.695960998535156 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.695816993713379 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.695936679840088 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.695989608764648 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.695908069610596 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.6958537101745605 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.695986270904541 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.695667743682861 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.695316791534424 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.6958842277526855 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.695745944976807 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.695991039276123 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.696067810058594 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.6956892013549805 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.6957855224609375 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.695777893066406 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.696026802062988 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.6959943771362305 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.69590425491333 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.6962080001831055 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.695856094360352 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.695924282073975 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.695370197296143 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.695986747741699 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.69608736038208 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.695954322814941 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.695711135864258 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.695876598358154 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.69609260559082 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.696167469024658 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.695797920227051 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.695643424987793 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.695591449737549 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.6958184242248535 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.696102142333984 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.695807456970215 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.695912837982178 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.695748805999756 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.695747375488281 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.6955790519714355 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.69586706161499 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.695943355560303 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.695437908172607 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.6956658363342285 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.695859432220459 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.695797920227051 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.69614839553833 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.695916175842285 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.696164608001709 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.695976257324219 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.695509433746338 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.69572114944458 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.695676803588867 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.695681571960449 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.695993900299072 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.695934295654297 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.695730686187744 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.69599723815918 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.695876598358154 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.6957316398620605 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.695810317993164 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.695830821990967 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.695494174957275 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.695385456085205 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.695666313171387 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.695861339569092 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.69589900970459 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.69561243057251 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.695858001708984 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.696011543273926 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.69523811340332 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.696177005767822 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.695779323577881 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.696069717407227 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.695669174194336 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.6957621574401855 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.6963701248168945 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.697138786315918 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.69717264175415 + } + ] + }, + { + "bit": 12, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.682405471801758 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.682546138763428 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.682572364807129 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.682584285736084 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.682428359985352 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.682394981384277 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.682276725769043 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.68253755569458 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.682480812072754 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.681424617767334 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.682178974151611 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.682028770446777 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.6821208000183105 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.6822509765625 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.682407379150391 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.68220329284668 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.682620525360107 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.682609558105469 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.682316780090332 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.682409763336182 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.682520866394043 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.682620525360107 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.682216167449951 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.6822190284729 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.682043075561523 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.68239688873291 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.682298183441162 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.6826300621032715 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.682465076446533 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.682571887969971 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.682335376739502 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.682166576385498 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.682760238647461 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.682681083679199 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.682453155517578 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.682560443878174 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.682487964630127 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.682333469390869 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.682610988616943 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.6823320388793945 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.682116508483887 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.68250036239624 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.682651996612549 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.682195663452148 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.682064533233643 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.682161808013916 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.682633399963379 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.683144569396973 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.682223320007324 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.682496070861816 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.682621002197266 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.681970119476318 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.68222188949585 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.682379722595215 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.682897090911865 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.682913303375244 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.68264102935791 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.682676315307617 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.681631565093994 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.682340145111084 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.682380676269531 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.682514667510986 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.682775020599365 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.682497978210449 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.682555675506592 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.681719779968262 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.682424068450928 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.682434558868408 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.682102680206299 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.682690620422363 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.682444095611572 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.682558536529541 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.682116508483887 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.681971549987793 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.682694435119629 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.683014869689941 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.6828413009643555 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.682681083679199 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.68265962600708 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.682471752166748 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.68213415145874 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.682506561279297 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.6824469566345215 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.682090759277344 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.682521820068359 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.682514667510986 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.681995391845703 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.682126045227051 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.682863235473633 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.68223762512207 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.682862281799316 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.68220329284668 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.682450294494629 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.682204723358154 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.682432174682617 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.6824421882629395 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.682489395141602 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.682785511016846 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.682525157928467 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.682422161102295 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.682413578033447 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.682464122772217 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.682589530944824 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.682800769805908 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.682434558868408 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.682404041290283 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.6823625564575195 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.682600021362305 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.68203067779541 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.6828742027282715 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.682847499847412 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.6824541091918945 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.682492733001709 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.68253231048584 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.682356834411621 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.682616710662842 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.682480812072754 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.6823811531066895 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.682729721069336 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.6825079917907715 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.682644367218018 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.682372570037842 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.6821489334106445 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.68222188949585 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.682405948638916 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.682449817657471 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.682379722595215 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.68241024017334 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.6825079917907715 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.682476997375488 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.6822662353515625 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.682485103607178 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.682605743408203 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.682283878326416 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.682496070861816 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.68239688873291 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.682328701019287 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.6824822425842285 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.682392120361328 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.682586669921875 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.682641506195068 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.682494640350342 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.68251895904541 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.682359218597412 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.68239688873291 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.682567596435547 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.682220935821533 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.682324409484863 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.682422637939453 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.682376384735107 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.682465076446533 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.682358264923096 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.682297229766846 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.682581901550293 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.682465076446533 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.682485580444336 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.682575702667236 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.682376384735107 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.682477951049805 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.682365894317627 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.682522296905518 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.682480812072754 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.682465076446533 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.682648658752441 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.682354927062988 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.682523250579834 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.682462215423584 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.682741165161133 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.682784557342529 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.682547569274902 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.68222188949585 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.682444095611572 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.68265962600708 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.682278633117676 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.682117462158203 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.682397842407227 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.682478904724121 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.682300567626953 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.682203769683838 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.682318687438965 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.682586669921875 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.682551383972168 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.682465076446533 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.682419776916504 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.682401180267334 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.682356834411621 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.682514667510986 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.682650566101074 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.682427406311035 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.682586669921875 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.682435035705566 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.682465076446533 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.682424068450928 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.682689666748047 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.682621479034424 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.682534217834473 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.682064056396484 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.682281970977783 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.682474136352539 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.6822919845581055 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.682399749755859 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.682446479797363 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.6823015213012695 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.682318210601807 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.682347774505615 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.682242393493652 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.6825270652771 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.682636737823486 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.682506561279297 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.682621479034424 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.6823859214782715 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.682356834411621 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.682347774505615 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.682248115539551 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.6826395988464355 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.681845664978027 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.682002067565918 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.682235240936279 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.682433605194092 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.682642459869385 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.682449817657471 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.683050155639648 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.683516502380371 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.682960510253906 + } + ] + }, + { + "bit": 13, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.675159931182861 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.6752119064331055 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.674989700317383 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.675139904022217 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.675213813781738 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.675201416015625 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.675170421600342 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.675137519836426 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.675296783447266 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.6738386154174805 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.675734519958496 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.675148010253906 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.674975395202637 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.6749420166015625 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.674900531768799 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.675200939178467 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.674682140350342 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.675135135650635 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.674946308135986 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.675005912780762 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.67481803894043 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.675193786621094 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.675198078155518 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.675240993499756 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.675490379333496 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.6749587059021 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.675299644470215 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.674860954284668 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.675156116485596 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.675148010253906 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.674904823303223 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.675342559814453 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.675131797790527 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.674973964691162 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.675352573394775 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.6752753257751465 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.675084114074707 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.674463272094727 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.675007343292236 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.675378799438477 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.675233364105225 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.675143241882324 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.675004005432129 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.675471782684326 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.674965858459473 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.6751627922058105 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.675118923187256 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.674881458282471 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.675345420837402 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.67519474029541 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.675193786621094 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.675271511077881 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.675358772277832 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.675508975982666 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.675199508666992 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.674704551696777 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.675106525421143 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.6751933097839355 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.675111293792725 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.674880027770996 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.675393581390381 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.675256252288818 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.674931049346924 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.675198078155518 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.675098896026611 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.675319671630859 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.675107002258301 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.67510986328125 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.674988746643066 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.6751861572265625 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.675061225891113 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.675054550170898 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.674835205078125 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.675243377685547 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.6749372482299805 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.674750804901123 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.674859046936035 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.675590515136719 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.675236225128174 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.6751861572265625 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.675496578216553 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.675129413604736 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.674994945526123 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.6752753257751465 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.675360202789307 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.675236225128174 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.675005912780762 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.675249099731445 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.675204753875732 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.675756931304932 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.674900054931641 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.675131797790527 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.675085544586182 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.674859046936035 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.674960613250732 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.675267219543457 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.675191879272461 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.6750288009643555 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.675195693969727 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.675181865692139 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.674865245819092 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.674902439117432 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.675182819366455 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.675046920776367 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.674983024597168 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.675196647644043 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.675022125244141 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.674625873565674 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.675236225128174 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.674773216247559 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.674907207489014 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.674955368041992 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.675133228302002 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.675105094909668 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.6748151779174805 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.675068378448486 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.6748576164245605 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.675285816192627 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.674724578857422 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.675057411193848 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.6751837730407715 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.674896240234375 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.675215721130371 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.675042629241943 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.6751484870910645 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.675325870513916 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.675143241882324 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.675173759460449 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.674859523773193 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.674994945526123 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.674944877624512 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.675033092498779 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.674886703491211 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.675206661224365 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.675133228302002 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.674929141998291 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.675138473510742 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.67514181137085 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.6750688552856445 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.674992084503174 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.675188064575195 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.675196647644043 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.674984455108643 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.675215721130371 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.675103664398193 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.674947738647461 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.675031661987305 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.675066947937012 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.675139904022217 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.6752095222473145 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.675175189971924 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.675001621246338 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.675168991088867 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.674901962280273 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.675100803375244 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.675065517425537 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.675075054168701 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.6750993728637695 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.674862384796143 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.6752190589904785 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.6751604080200195 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.6752519607543945 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.675191879272461 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.675068378448486 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.675307273864746 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.674890995025635 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.675022125244141 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.6750688552856445 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.675222873687744 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.675219535827637 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.675410747528076 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.675120830535889 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.675173759460449 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.675325393676758 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.675308704376221 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.675172805786133 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.6748504638671875 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.674747467041016 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.675097942352295 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.6753644943237305 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.674871921539307 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.674992084503174 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.675086975097656 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.674976825714111 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.674874782562256 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.6751251220703125 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.675076007843018 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.675105094909668 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.6750407218933105 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.675085544586182 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.674973487854004 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.67523193359375 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.675292015075684 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.675056457519531 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.675045490264893 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.674822807312012 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.674994945526123 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.674855709075928 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.674783229827881 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.675138473510742 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.675293445587158 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.675244331359863 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.675158977508545 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.675165176391602 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.6750712394714355 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.675082683563232 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.675072193145752 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.674965858459473 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.6751508712768555 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.675063610076904 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.6750807762146 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.675137042999268 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.6748833656311035 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.675176620483398 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.674953937530518 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.6749267578125 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.675621509552002 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.675137042999268 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.675126552581787 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.6749138832092285 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.674839496612549 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.6758928298950195 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.675153732299805 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.676681995391846 + } + ] + }, + { + "bit": 14, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.667819976806641 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.667902946472168 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.667791366577148 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.667810916900635 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.667779445648193 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.667750358581543 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.667586803436279 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.667886257171631 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.667913436889648 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.666787624359131 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.6674675941467285 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.667463302612305 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.667484283447266 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.667829990386963 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.667748928070068 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.667736530303955 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.667854309082031 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.667791843414307 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.667515277862549 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.66773796081543 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.6678571701049805 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.667884349822998 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.667843341827393 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.667605400085449 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.66740083694458 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.66780948638916 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.667678356170654 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.667628288269043 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.667821884155273 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.66787576675415 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.667562484741211 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.6675124168396 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.667896747589111 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.668192386627197 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.667636871337891 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.6678266525268555 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.667893409729004 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.667824745178223 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.667710781097412 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.667720794677734 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.667701721191406 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.667393207550049 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.668048858642578 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.667686462402344 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.66743803024292 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.667823314666748 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.668063640594482 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.668030738830566 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.6678080558776855 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.667759418487549 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.6677961349487305 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.667259693145752 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.667713165283203 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.667640686035156 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.667692184448242 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.667765140533447 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.668081760406494 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.667988300323486 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.667435646057129 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.6680216789245605 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.667642116546631 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.6677727699279785 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.668337821960449 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.667844772338867 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.667926788330078 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.667381286621094 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.667594909667969 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.667700290679932 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.6677069664001465 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.667908668518066 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.667959690093994 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.667820453643799 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.666912078857422 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.66742467880249 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.667907238006592 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.668238162994385 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.668012619018555 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.667900085449219 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.667880535125732 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.667731285095215 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.667665004730225 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.667799472808838 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.667914390563965 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.6679840087890625 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.667851448059082 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.667998790740967 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.667558193206787 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.667721748352051 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.66801118850708 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.667507171630859 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.668368339538574 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.667765140533447 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.6675567626953125 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.667446136474609 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.667893409729004 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.667877197265625 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.667776107788086 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.668105602264404 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.6677565574646 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.667980194091797 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.66771936416626 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.668062686920166 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.667606353759766 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.667844772338867 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.66787052154541 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.6678900718688965 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.66793155670166 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.668067932128906 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.6676554679870605 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.668066501617432 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.6683759689331055 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.66816520690918 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.667838096618652 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.667967796325684 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.66756534576416 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.667832374572754 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.667873859405518 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.667971611022949 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.668344974517822 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.667962074279785 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.6679229736328125 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.667703151702881 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.667819023132324 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.667728424072266 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.667746067047119 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.667822360992432 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.667943477630615 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.667981147766113 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.668025016784668 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.667801856994629 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.667850971221924 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.667886257171631 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.667900085449219 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.667832851409912 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.668010234832764 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.667873382568359 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.668032169342041 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.6679229736328125 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.66804313659668 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.667966365814209 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.66804313659668 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.667793273925781 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.667874336242676 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.667849063873291 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.667914867401123 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.667708396911621 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.667759418487549 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.667840003967285 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.667886257171631 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.667885780334473 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.6679229736328125 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.66795015335083 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.667733669281006 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.667864799499512 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.667882919311523 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.667743682861328 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.667877197265625 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.6678876876831055 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.668033123016357 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.6679606437683105 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.66779088973999 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.667966365814209 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.667866230010986 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.668037414550781 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.667810916900635 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.667989730834961 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.6678786277771 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.667901515960693 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.668017387390137 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.66794490814209 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.667586803436279 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.667880058288574 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.667779445648193 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.66774845123291 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.6673760414123535 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.667710781097412 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.667843341827393 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.667978763580322 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.667820453643799 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.667684078216553 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.667862892150879 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.667961120605469 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.667817115783691 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.667698383331299 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.66771936416626 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.667913436889648 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.6679487228393555 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.667896270751953 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.667839050292969 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.667908668518066 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.667864799499512 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.667849063873291 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.667857646942139 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.6679205894470215 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.668027877807617 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.668027877807617 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.6677327156066895 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.667728424072266 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.667996883392334 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.667733192443848 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.667900085449219 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.66773796081543 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.667776584625244 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.667733669281006 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.667789459228516 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.667705535888672 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.668050289154053 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.667913436889648 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.667654991149902 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.667719841003418 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.667740821838379 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.667862892150879 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.667603492736816 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.667653560638428 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.667785167694092 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.6674065589904785 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.66767692565918 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.667727470397949 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.667888641357422 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.668050289154053 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.6679606437683105 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.6680908203125 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.669186592102051 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.6680426597595215 + } + ] + }, + { + "bit": 15, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.655381679534912 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.655350208282471 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.655416488647461 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.655372142791748 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.655397415161133 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.655365467071533 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.655310153961182 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.655349254608154 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.655430316925049 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.654942035675049 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.655637741088867 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.655438423156738 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.655168533325195 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.655331611633301 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.65532112121582 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.655498027801514 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.655391216278076 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.655319690704346 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.655374526977539 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.655303955078125 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.6550211906433105 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.65546989440918 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.655459403991699 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.655435562133789 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.655843734741211 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.6554341316223145 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.655528545379639 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.655211448669434 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.655364036560059 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.655369758605957 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.655393600463867 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.655636310577393 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.6551971435546875 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.655117034912109 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.6555938720703125 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.655561447143555 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.655438423156738 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.654885768890381 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.655267238616943 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.655786037445068 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.655411243438721 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.655387878417969 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.655194282531738 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.655491828918457 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.655220985412598 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.655439853668213 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.655453205108643 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.655322551727295 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.65521240234375 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.6554059982299805 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.655473709106445 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.6555705070495605 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.6552886962890625 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.655696868896484 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.655643939971924 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.655392646789551 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.6555352210998535 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.6553425788879395 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.655293941497803 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.655301094055176 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.655514717102051 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.655604362487793 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.655079364776611 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.655487060546875 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.655508995056152 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.655398368835449 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.655621528625488 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.655509948730469 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.655366897583008 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.655523300170898 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.655355930328369 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.655323028564453 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.655409812927246 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.655735015869141 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.655531406402588 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.655338287353516 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.655355453491211 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.655698299407959 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.655580043792725 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.655551433563232 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.655698299407959 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.655328750610352 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.655311584472656 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.655616283416748 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.655621528625488 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.65546178817749 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.655242919921875 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.655585289001465 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.655377388000488 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.655829429626465 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.655137062072754 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.655309200286865 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.655547618865967 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.655132293701172 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.6552734375 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.655489921569824 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.655557155609131 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.655264377593994 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.655502796173096 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.65546178817749 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.655019760131836 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.65524959564209 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.655608654022217 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.655491828918457 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.655444145202637 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.655490875244141 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.655350208282471 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.655115604400635 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.65542459487915 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.655378818511963 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.655139446258545 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.655368328094482 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.6554274559021 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.655401706695557 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.65524959564209 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.65509557723999 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.655533790588379 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.655591011047363 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.655198097229004 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.65537691116333 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.655484199523926 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.655308246612549 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.655356407165527 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.655351161956787 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.655346870422363 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.655555725097656 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.655323505401611 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.655442237854004 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.655481338500977 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.655405044555664 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.655328273773193 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.655386924743652 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.655391216278076 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.655398845672607 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.6554460525512695 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.655357360839844 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.655369758605957 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.655479907989502 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.655420303344727 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.655256748199463 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.655384540557861 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.655562877655029 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.655280113220215 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.655511379241943 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.655328750610352 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.655407428741455 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.655326843261719 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.655528545379639 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.655569553375244 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.655414581298828 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.655400276184082 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.655462741851807 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.655518531799316 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.655440330505371 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.655346870422363 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.655397415161133 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.655476093292236 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.655486106872559 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.655329704284668 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.655313014984131 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.655557632446289 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.65554666519165 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.65546989440918 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.655310153961182 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.655532360076904 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.655371189117432 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.655349254608154 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.655484199523926 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.655538082122803 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.655428886413574 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.655603408813477 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.655642509460449 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.655392646789551 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.655400276184082 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.655543327331543 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.6554412841796875 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.655256748199463 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.655172824859619 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.655358791351318 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.655686378479004 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.655426025390625 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.655337333679199 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.655442237854004 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.655550003051758 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.655192852020264 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.655454635620117 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.655520915985107 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.655393123626709 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.655390739440918 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.655576705932617 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.655282974243164 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.655608177185059 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.6555070877075195 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.655572414398193 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.655426025390625 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.65526819229126 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.655401706695557 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.655243873596191 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.655099868774414 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.655643463134766 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.655508995056152 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.655423164367676 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.655397415161133 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.655409812927246 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.655454635620117 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.655190944671631 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.655305862426758 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.655282974243164 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.655362606048584 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.655386924743652 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.655330181121826 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.655435085296631 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.65518856048584 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.655323028564453 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.655422687530518 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.655121803283691 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.6556243896484375 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.655403137207031 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.655536651611328 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.65526819229126 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.655147075653076 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.656041145324707 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.6555352210998535 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.656767845153809 + } + ] + } + ] +} \ No newline at end of file